webClientProvider,
+ Logger log,
+ CrawlCacheService crawlCacheService,
+ @ServiceKey String key) {
+ super(log, key, crawlCacheService);
+
+ this.webClientProvider = webClientProvider;
+ }
+
+ @Override
+ protected WebClient getWebClient() {
+ return webClientProvider.get();
+ }
+
+ @Override
+ public long getCachedPageTimeoutSec() {
+ return cachedPageTimeoutSec;
+ }
+
+ @Override
+ public long getTimeoutMillis() {
+ return timeoutMillis;
+ }
+}
diff --git a/gwtp-crawler-service/src/main/java/com/gwtplatform/crawlerservice/server/HtmlUnitTimeoutMillis.java b/gwtp-core/gwtp-crawler-guice/src/main/java/com/gwtplatform/crawler/server/guice/service/HtmlUnitTimeoutMillis.java
similarity index 96%
rename from gwtp-crawler-service/src/main/java/com/gwtplatform/crawlerservice/server/HtmlUnitTimeoutMillis.java
rename to gwtp-core/gwtp-crawler-guice/src/main/java/com/gwtplatform/crawler/server/guice/service/HtmlUnitTimeoutMillis.java
index 39800076b0..ad488ae47d 100644
--- a/gwtp-crawler-service/src/main/java/com/gwtplatform/crawlerservice/server/HtmlUnitTimeoutMillis.java
+++ b/gwtp-core/gwtp-crawler-guice/src/main/java/com/gwtplatform/crawler/server/guice/service/HtmlUnitTimeoutMillis.java
@@ -14,7 +14,7 @@
* the License.
*/
-package com.gwtplatform.crawlerservice.server;
+package com.gwtplatform.crawler.server.guice.service;
import java.lang.annotation.Retention;
import java.lang.annotation.Target;
diff --git a/gwtp-core/gwtp-crawler-spring/pom.xml b/gwtp-core/gwtp-crawler-spring/pom.xml
new file mode 100644
index 0000000000..8541f4f734
--- /dev/null
+++ b/gwtp-core/gwtp-crawler-spring/pom.xml
@@ -0,0 +1,73 @@
+
+
+ 4.0.0
+
+
+ com.gwtplatform
+ gwtp-core
+ 1.6-SNAPSHOT
+
+
+ gwtp-crawler-spring
+ GWTP Crawler for Spring
+
+
+
+
+
+ src/main/java
+
+ **/*.java
+
+
+
+
+ src/main/resources
+
+ **/*.gwt.xml
+
+
+
+
+
+
+
+ ${project.groupId}
+ gwtp-crawler
+
+
+
+ javax.servlet
+ servlet-api
+
+
+
+ org.springframework
+ spring-core
+ ${spring.version}
+
+
+ org.springframework
+ spring-context
+ ${spring.version}
+
+
+ org.springframework
+ spring-beans
+ ${spring.version}
+
+
+ org.springframework
+ spring-web
+ ${spring.version}
+
+
+
+
+ org.springframework
+ spring-test
+ ${spring.version}
+ test
+
+
+
diff --git a/gwtp-core/gwtp-crawler-spring/src/main/java/com/gwtplatform/crawler/server/spring/AbstractCrawlerModule.java b/gwtp-core/gwtp-crawler-spring/src/main/java/com/gwtplatform/crawler/server/spring/AbstractCrawlerModule.java
new file mode 100644
index 0000000000..9317e1ef31
--- /dev/null
+++ b/gwtp-core/gwtp-crawler-spring/src/main/java/com/gwtplatform/crawler/server/spring/AbstractCrawlerModule.java
@@ -0,0 +1,35 @@
+/*
+ * Copyright 2015 ArcBees Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package com.gwtplatform.crawler.server.spring;
+
+import java.util.logging.Logger;
+
+import org.springframework.context.annotation.Bean;
+
+/**
+ * Abstract crawler module for {@link @Configuration} setup.
+ * @author Ben Dol
+ */
+public abstract class AbstractCrawlerModule {
+ @Bean
+ protected Logger crawlLogger() {
+ return Logger.getAnonymousLogger();
+ }
+
+ @Bean
+ protected abstract String crawlKey();
+}
diff --git a/gwtp-core/gwtp-crawler-spring/src/main/java/com/gwtplatform/crawler/server/spring/filter/AbstractCrawlFilterModule.java b/gwtp-core/gwtp-crawler-spring/src/main/java/com/gwtplatform/crawler/server/spring/filter/AbstractCrawlFilterModule.java
new file mode 100644
index 0000000000..d0dc8053f8
--- /dev/null
+++ b/gwtp-core/gwtp-crawler-spring/src/main/java/com/gwtplatform/crawler/server/spring/filter/AbstractCrawlFilterModule.java
@@ -0,0 +1,35 @@
+/*
+ * Copyright 2015 ArcBees Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package com.gwtplatform.crawler.server.spring.filter;
+
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.ComponentScan;
+
+import com.gwtplatform.crawler.server.spring.AbstractCrawlerModule;
+
+/**
+ * Abstract crawl filter module for {@link @Configuration} setup.
+ * @author Ben Dol
+ */
+@ComponentScan(basePackages = {
+ "com.gwtplatform.crawler.server.spring.filter"
+ })
+public abstract class AbstractCrawlFilterModule extends AbstractCrawlerModule {
+
+ @Bean
+ protected abstract String serviceUrl();
+}
diff --git a/gwtp-core/gwtp-crawler-spring/src/main/java/com/gwtplatform/crawler/server/spring/filter/SpringCrawlFilter.java b/gwtp-core/gwtp-crawler-spring/src/main/java/com/gwtplatform/crawler/server/spring/filter/SpringCrawlFilter.java
new file mode 100644
index 0000000000..c07a79527b
--- /dev/null
+++ b/gwtp-core/gwtp-crawler-spring/src/main/java/com/gwtplatform/crawler/server/spring/filter/SpringCrawlFilter.java
@@ -0,0 +1,65 @@
+/*
+ * Copyright 2015 ArcBees Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package com.gwtplatform.crawler.server.spring.filter;
+
+import java.util.logging.Logger;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+import org.springframework.web.WebApplicationInitializer;
+
+import com.gwtplatform.crawler.server.CrawlFilter;
+
+/**
+ * Spring implementation for the {@link CrawlFilter}.
+ * Required bean dependencies are:
+ *
+ * - serviceUrl (String): Url for the crawler service.
+ * - crawlKey (String): Unique key for the crawler service.
+ * - crawlLogger (Logger): Logger for the crawl filter.
+ *
+ * Extend the {@link AbstractCrawlFilterModule} with
+ * {@link org.springframework.beans.factory.annotation.Configurable} class.
+ *
+ * Then register inside web.xml like so:
+ *
+ * {@code
+ *
+ * crawlFilter
+ * org.springframework.web.filter.DelegatingFilterProxy
+ *
+ *
+ * crawlFilter
+ * /*
+ * }
+ *
+ * or instead using {@link WebApplicationInitializer}:
+ *
+ * servletContext.addFilter("crawlFilter", new DelegatingFilterProxy())
+ * .addMappingForUrlPatterns(EnumSet.of(DispatcherType.REQUEST), true, "/*");
+ *
+ *
+ * @author Ben Dol
+ */
+@Component("crawlFilter")
+public final class SpringCrawlFilter extends CrawlFilter {
+
+ @Autowired
+ SpringCrawlFilter(String serviceUrl, String crawlKey, Logger crawlLogger) {
+ super(serviceUrl, crawlKey, crawlLogger);
+ }
+}
diff --git a/gwtp-core/gwtp-crawler-spring/src/main/java/com/gwtplatform/crawler/server/spring/service/AbstractCrawlServiceModule.java b/gwtp-core/gwtp-crawler-spring/src/main/java/com/gwtplatform/crawler/server/spring/service/AbstractCrawlServiceModule.java
new file mode 100644
index 0000000000..77be5ab0e7
--- /dev/null
+++ b/gwtp-core/gwtp-crawler-spring/src/main/java/com/gwtplatform/crawler/server/spring/service/AbstractCrawlServiceModule.java
@@ -0,0 +1,45 @@
+/*
+ * Copyright 2015 ArcBees Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package com.gwtplatform.crawler.server.spring.service;
+
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.ComponentScan;
+
+import com.gargoylesoftware.htmlunit.BrowserVersion;
+import com.gargoylesoftware.htmlunit.WebClient;
+import com.gwtplatform.crawler.server.CrawlCacheService;
+import com.gwtplatform.crawler.server.DefaultCrawlCacheService;
+import com.gwtplatform.crawler.server.spring.AbstractCrawlerModule;
+
+/**
+ * Abstract crawl service module for {@link @Configuration} setup.
+ * @author Ben Dol
+ */
+@ComponentScan(basePackages = {
+ "com.gwtplatform.crawler.server.spring.service"
+ })
+public abstract class AbstractCrawlServiceModule extends AbstractCrawlerModule {
+ @Bean
+ protected WebClient webClient() {
+ return new WebClient(BrowserVersion.CHROME);
+ }
+
+ @Bean
+ protected CrawlCacheService crawlCacheService() {
+ return new DefaultCrawlCacheService();
+ }
+}
diff --git a/gwtp-core/gwtp-crawler-spring/src/main/java/com/gwtplatform/crawler/server/spring/service/CrawlServiceServlet.java b/gwtp-core/gwtp-crawler-spring/src/main/java/com/gwtplatform/crawler/server/spring/service/CrawlServiceServlet.java
new file mode 100644
index 0000000000..fcc75db812
--- /dev/null
+++ b/gwtp-core/gwtp-crawler-spring/src/main/java/com/gwtplatform/crawler/server/spring/service/CrawlServiceServlet.java
@@ -0,0 +1,122 @@
+/*
+ * Copyright 2015 ArcBees Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package com.gwtplatform.crawler.server.spring.service;
+
+import java.io.IOException;
+import java.util.logging.Logger;
+
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+import org.springframework.web.HttpRequestHandler;
+import org.springframework.web.WebApplicationInitializer;
+
+import com.gargoylesoftware.htmlunit.WebClient;
+import com.gwtplatform.crawler.server.AbstractCrawlServiceServlet;
+import com.gwtplatform.crawler.server.CrawlCacheService;
+import com.gwtplatform.crawler.server.CrawledPage;
+
+/**
+ * Spring Crawl Service Servlet.
+ * Required bean dependencies are:
+ *
+ * - webClient ({@link WebClient}): HTML Unit virtual web client.
+ * - crawlCacheService ({@link CrawlCacheService}): Crawled page cache service.
+ * - crawlKey (String): Unique key for the crawler service.
+ * - crawlLogger (Logger): Logger for the crawl filter.
+ * - timeoutMillis (long:5000): The HTML Unit Timeout in milliseconds.
+ * - cachedPageTimeoutSec (long:900): Cache timeout period before {@link CrawledPage}'s are invalidated.
+ *
+ * Extend the {@link AbstractCrawlServiceModule} with
+ * {@link org.springframework.beans.factory.annotation.Configurable} class.
+ *
+ * Then register in web.xml like so:
+ *
+ * {@code
+ * <-- First ensure you have the ContextLoaderListener -->
+ *
+ * org.springframework.web.context.ContextLoaderListener
+ *
+ *
+ *
+ * crawlServiceServlet
+ * org.springframework.web.context.support.HttpRequestHandlerServlet
+ *
+ *
+ *
+ * crawlServiceServlet
+ * /*
+ * }
+ *
+ * or using {@link WebApplicationInitializer}:
+ *
+ * // Ensure you have registered the ContextLoaderListener
+ * servletContext.addListener(new ContextLoaderListener(context));
+ *
+ * // Register the new servlet as a HttpRequestHandlerServlet
+ * servletContext.addServlet("crawlServiceServlet", new HttpRequestHandlerServlet()).addMapping("/*");
+ *
+ *
+ * @author Ben Dol
+ */
+@Component
+public class CrawlServiceServlet extends AbstractCrawlServiceServlet implements HttpRequestHandler {
+
+ @Value("${timeoutMillis:5000}")
+ private long timeoutMillis;
+
+ @Value("${cachedPageTimeoutSec:900}")
+ private long cachedPageTimeoutSec;
+
+ private final WebClient webClient;
+
+ @Autowired
+ protected CrawlServiceServlet(
+ WebClient webClient,
+ CrawlCacheService crawlCacheService,
+ String crawlKey,
+ Logger crawlLogger) {
+ super(crawlLogger, crawlKey, crawlCacheService);
+
+ this.webClient = webClient;
+ }
+
+ @Override
+ public void handleRequest(HttpServletRequest request, HttpServletResponse response)
+ throws ServletException, IOException {
+ doGet(request, response);
+ }
+
+ @Override
+ protected WebClient getWebClient() {
+ return webClient;
+ }
+
+ @Override
+ public long getCachedPageTimeoutSec() {
+ return cachedPageTimeoutSec;
+ }
+
+ @Override
+ public long getTimeoutMillis() {
+ return timeoutMillis;
+ }
+}
diff --git a/gwtp-core/gwtp-crawler/pom.xml b/gwtp-core/gwtp-crawler/pom.xml
index 6ccb64a9b8..8202f28992 100644
--- a/gwtp-core/gwtp-crawler/pom.xml
+++ b/gwtp-core/gwtp-crawler/pom.xml
@@ -36,10 +36,9 @@
servlet-api
-
- com.google.inject
- guice
+ net.sourceforge.htmlunit
+ htmlunit
diff --git a/gwtp-crawler-service/src/main/java/com/gwtplatform/crawlerservice/server/CrawlServiceServlet.java b/gwtp-core/gwtp-crawler/src/main/java/com/gwtplatform/crawler/server/AbstractCrawlServiceServlet.java
similarity index 70%
rename from gwtp-crawler-service/src/main/java/com/gwtplatform/crawlerservice/server/CrawlServiceServlet.java
rename to gwtp-core/gwtp-crawler/src/main/java/com/gwtplatform/crawler/server/AbstractCrawlServiceServlet.java
index a45b95a9b7..a4836f7c2e 100644
--- a/gwtp-crawler-service/src/main/java/com/gwtplatform/crawlerservice/server/CrawlServiceServlet.java
+++ b/gwtp-core/gwtp-crawler/src/main/java/com/gwtplatform/crawler/server/AbstractCrawlServiceServlet.java
@@ -14,7 +14,7 @@
* the License.
*/
-package com.gwtplatform.crawlerservice.server;
+package com.gwtplatform.crawler.server;
import java.io.IOException;
import java.io.PrintWriter;
@@ -26,8 +26,6 @@
import java.util.logging.Logger;
import java.util.regex.Pattern;
-import javax.inject.Provider;
-import javax.inject.Singleton;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
@@ -37,17 +35,13 @@
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.WebRequest;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
-import com.google.common.base.Strings;
-import com.google.inject.Inject;
-import com.googlecode.objectify.Key;
-import com.gwtplatform.crawlerservice.server.domain.CachedPage;
-import com.gwtplatform.crawlerservice.server.service.CachedPageDao;
/**
- * Servlet that makes it possible to fetch an external page, renders it using HTMLUnit and returns the HTML page.
+ * Servlet that makes it possible to fetch an external page,
+ * renders it using HTMLUnit and returns the HTML page.
*/
-@Singleton
-public class CrawlServiceServlet extends HttpServlet {
+@SuppressWarnings("unchecked")
+public abstract class AbstractCrawlServiceServlet extends HttpServlet {
private static class SyncAllAjaxController extends NicelyResynchronizingAjaxController {
private static final long serialVersionUID = 1L;
@@ -58,40 +52,37 @@ public boolean processSynchron(HtmlPage page, WebRequest request, boolean async)
}
}
- private static final String CHAR_ENCODING = "UTF-8";
+ protected static final String CHAR_ENCODING = "UTF-8";
private static final long serialVersionUID = -6129110224710383122L;
- @Inject(optional = true)
- @HtmlUnitTimeoutMillis
- private final long timeoutMillis = 5000;
- private final long jsTimeoutMillis = 2000;
- private final long pageWaitMillis = 100;
- private final long maxLoopChecks = 2;
+ protected final Logger log;
+ protected final String key;
- @Inject(optional = true)
- @CachedPageTimeoutSec
- private final long cachedPageTimeoutSec = 15 * 60;
+ private final CrawlCacheService cacheService;
- private final Logger log;
- private final Provider webClientProvider;
-
- private final String key;
-
- private final CachedPageDao cachedPageDao;
+ public AbstractCrawlServiceServlet(
+ Logger log,
+ String key) {
+ this(log, key, null);
+ }
- @Inject
- protected CrawlServiceServlet(
- Provider webClientProvider,
+ public AbstractCrawlServiceServlet(
Logger log,
- CachedPageDao cachedPageDao,
- @ServiceKey String key) {
- this.webClientProvider = webClientProvider;
+ String key,
+ CrawlCacheService cacheService) {
this.log = log;
this.key = key;
- this.cachedPageDao = cachedPageDao;
+
+ if (cacheService != null) {
+ this.cacheService = cacheService;
+ } else {
+ this.cacheService = new DefaultCrawlCacheService();
+ }
}
+ protected abstract WebClient getWebClient();
+
@Override
protected void doGet(HttpServletRequest request, HttpServletResponse response) {
PrintWriter out = null;
@@ -104,17 +95,17 @@ protected void doGet(HttpServletRequest request, HttpServletResponse response) {
validateKey(request);
String url = request.getParameter("url");
- if (!Strings.isNullOrEmpty(url)) {
+ if (url != null && !url.isEmpty()) {
url = URLDecoder.decode(url, CHAR_ENCODING);
- CachedPage cachedPage = cachedPageDao.get(Key.create(CachedPage.class, url));
+ CrawledPage crawledPage = cacheService.getCachedPage(url);
Date currDate = new Date();
- if (needToFetchPage(cachedPage, currDate, out)) {
- cachedPage = createPlaceholderPage(url, currDate);
+ if (needToFetchPage(crawledPage, currDate, out)) {
+ crawledPage = createPlaceholderPage(url, currDate);
String renderedHtml = renderPage(url);
- storeFetchedPage(cachedPage, renderedHtml);
+ storeFetchedPage(crawledPage, renderedHtml);
out.println(renderedHtml);
}
}
@@ -135,7 +126,7 @@ private void validateKey(HttpServletRequest request)
throws InvalidKeyException, UnsupportedEncodingException {
String receivedKey = request.getParameter("key");
- if (Strings.isNullOrEmpty(receivedKey)) {
+ if (receivedKey == null || receivedKey.isEmpty()) {
throw new InvalidKeyException("No service key attached to the request.");
} else {
String decodedKey = URLDecoder.decode(receivedKey, CHAR_ENCODING);
@@ -146,10 +137,10 @@ private void validateKey(HttpServletRequest request)
}
}
- private void storeFetchedPage(CachedPage cachedPage, String stringBuilder) {
- cachedPage.setContent(stringBuilder);
- cachedPage.setFetchInProgress(false);
- cachedPageDao.put(cachedPage);
+ private void storeFetchedPage(CrawledPage crawledPage, String stringBuilder) {
+ crawledPage.setContent(stringBuilder);
+ crawledPage.setFetchInProgress(false);
+ cacheService.saveCachedPage(crawledPage);
}
/**
@@ -161,15 +152,15 @@ private void storeFetchedPage(CachedPage cachedPage, String stringBuilder) {
* @param out The {@link PrintWriter} to write to, if needed.
* @return {@code true} if the page needs to be fetched, {@code false} otherwise.
*/
- private boolean needToFetchPage(CachedPage matchingPage, Date currDate, PrintWriter out) {
- if (matchingPage == null || matchingPage.isExpired(cachedPageTimeoutSec)) {
+ private boolean needToFetchPage(CrawledPage matchingPage, Date currDate, PrintWriter out) {
+ if (matchingPage == null || matchingPage.isExpired(getCachedPageTimeoutSec())) {
return true;
}
if (matchingPage.isFetchInProgress()) {
// If fetch is in progress since more than 60 seconds, we consider something went wrong and fetch again.
if (currDate.getTime() > matchingPage.getFetchDate().getTime() + 60000) {
- cachedPageDao.delete(matchingPage);
+ cacheService.deleteCachedPage(matchingPage);
return true;
} else {
out.println("FETCH_IN_PROGRESS");
@@ -188,12 +179,12 @@ private boolean needToFetchPage(CachedPage matchingPage, Date currDate, PrintWri
* @param currDate The current date, to mark the page.
* @return The newly created placeholder page.
*/
- private CachedPage createPlaceholderPage(String url, Date currDate) {
- CachedPage result = new CachedPage();
+ private CrawledPage createPlaceholderPage(String url, Date currDate) {
+ CrawledPage result = cacheService.createCrawledPage();
result.setUrl(url);
result.setFetchDate(currDate);
result.setFetchInProgress(true);
- cachedPageDao.put(result);
+ cacheService.saveCachedPage(result);
return result;
}
@@ -205,7 +196,7 @@ private CachedPage createPlaceholderPage(String url, Date currDate) {
* @return The rendered page, in a {@link StringBuilder}.
*/
private String renderPage(String url) throws IOException {
- WebClient webClient = webClientProvider.get();
+ WebClient webClient = getWebClient();
webClient.getCache().clear();
webClient.getOptions().setCssEnabled(false);
@@ -218,12 +209,13 @@ private String renderPage(String url) throws IOException {
WebRequest webRequest = new WebRequest(new URL(url), "text/html");
HtmlPage page = webClient.getPage(webRequest);
- webClient.getJavaScriptEngine().pumpEventLoop(timeoutMillis);
+ webClient.getJavaScriptEngine().pumpEventLoop(getTimeoutMillis());
+ long jsTimeoutMillis = getJsTimeoutMillis();
int waitForBackgroundJavaScript = webClient.waitForBackgroundJavaScript(jsTimeoutMillis);
int loopCount = 0;
- while (waitForBackgroundJavaScript > 0 && loopCount < maxLoopChecks) {
+ while (waitForBackgroundJavaScript > 0 && loopCount < getMaxLoopChecks()) {
++loopCount;
waitForBackgroundJavaScript = webClient.waitForBackgroundJavaScript(jsTimeoutMillis);
@@ -235,7 +227,7 @@ private String renderPage(String url) throws IOException {
synchronized (page) {
log.fine("HtmlUnit waits for background javascript at loop counter " + loopCount);
try {
- page.wait(pageWaitMillis);
+ page.wait(getPageWaitMillis());
} catch (InterruptedException e) {
log.log(Level.SEVERE, "HtmlUnit ERROR on page.wait at loop counter " + loopCount, e);
}
@@ -248,4 +240,40 @@ private String renderPage(String url) throws IOException {
.matcher(page.asXml().replace("", ""))
.replaceAll("");
}
+
+ /**
+ * The HTML Unit Timeout in milliseconds.
+ */
+ public long getTimeoutMillis() {
+ return 5000;
+ }
+
+ /**
+ * The JavaScript load timeout in milliseconds.
+ */
+ public long getJsTimeoutMillis() {
+ return 2000;
+ }
+
+ /**
+ * Max page wait time in milliseconds.
+ */
+ public long getPageWaitMillis() {
+ return 100;
+ }
+
+ /**
+ * Max loop check value.
+ */
+ public long getMaxLoopChecks() {
+ return 2;
+ }
+
+ /**
+ * Cache timeout period before {@link CrawledPage}'s are invalidated.
+ * @return timeout period in seconds.
+ */
+ public long getCachedPageTimeoutSec() {
+ return 15 * 60;
+ }
}
diff --git a/gwtp-core/gwtp-crawler/src/main/java/com/gwtplatform/crawler/server/CrawlCacheService.java b/gwtp-core/gwtp-crawler/src/main/java/com/gwtplatform/crawler/server/CrawlCacheService.java
new file mode 100644
index 0000000000..6552c9442b
--- /dev/null
+++ b/gwtp-core/gwtp-crawler/src/main/java/com/gwtplatform/crawler/server/CrawlCacheService.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright 2015 ArcBees Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package com.gwtplatform.crawler.server;
+
+/**
+ * Crawl cache service interface.
+ * @author Ben Dol
+ */
+public interface CrawlCacheService {
+
+ T createCrawledPage();
+
+ T getCachedPage(String url);
+
+ void saveCachedPage(T crawledPage);
+
+ void deleteCachedPage(T crawledPage);
+}
diff --git a/gwtp-core/gwtp-crawler/src/main/java/com/gwtplatform/crawler/server/CrawlFilter.java b/gwtp-core/gwtp-crawler/src/main/java/com/gwtplatform/crawler/server/CrawlFilter.java
index 306d6709a3..390bc605e6 100644
--- a/gwtp-core/gwtp-crawler/src/main/java/com/gwtplatform/crawler/server/CrawlFilter.java
+++ b/gwtp-core/gwtp-crawler/src/main/java/com/gwtplatform/crawler/server/CrawlFilter.java
@@ -28,8 +28,6 @@
import java.net.URLEncoder;
import java.util.logging.Logger;
-import javax.inject.Inject;
-import javax.inject.Singleton;
import javax.servlet.Filter;
import javax.servlet.FilterChain;
import javax.servlet.FilterConfig;
@@ -40,10 +38,9 @@
import javax.servlet.http.HttpServletResponse;
/**
- * Servlet that makes this application crawlable.
+ * Servlet filter that makes this application crawlable.
*/
-@Singleton
-public final class CrawlFilter implements Filter {
+public class CrawlFilter implements Filter {
private static final String CHAR_ENCODING = "UTF-8";
/**
@@ -64,10 +61,7 @@ public final class CrawlFilter implements Filter {
private final String key;
private final Logger log;
- @Inject
- CrawlFilter(@ServiceUrl String serviceUrl,
- @ServiceKey String key,
- Logger log) {
+ protected CrawlFilter(String serviceUrl, String key, Logger log) {
this.serviceUrl = serviceUrl;
this.key = key;
this.log = log;
@@ -202,4 +196,12 @@ public void doFilter(ServletRequest request, ServletResponse response,
@Override
public void init(FilterConfig filterConfig) throws ServletException {
}
+
+ public String getServiceUrl() {
+ return serviceUrl;
+ }
+
+ public String getKey() {
+ return key;
+ }
}
diff --git a/gwtp-core/gwtp-crawler/src/main/java/com/gwtplatform/crawler/server/CrawledPage.java b/gwtp-core/gwtp-crawler/src/main/java/com/gwtplatform/crawler/server/CrawledPage.java
new file mode 100644
index 0000000000..36c50913cd
--- /dev/null
+++ b/gwtp-core/gwtp-crawler/src/main/java/com/gwtplatform/crawler/server/CrawledPage.java
@@ -0,0 +1,44 @@
+/*
+ * Copyright 2015 ArcBees Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package com.gwtplatform.crawler.server;
+
+import java.util.Date;
+
+/**
+ * Crawled page interface.
+ *
+ * @author Ben Dol
+ */
+public interface CrawledPage {
+ void setUrl(String url);
+
+ String getUrl();
+
+ void setFetchDate(Date fetchDate);
+
+ Date getFetchDate();
+
+ void setFetchInProgress(boolean fetchInProgress);
+
+ boolean isFetchInProgress();
+
+ void setContent(String content);
+
+ String getContent();
+
+ boolean isExpired(long cachedPageTimeoutSec);
+}
diff --git a/gwtp-core/gwtp-crawler/src/main/java/com/gwtplatform/crawler/server/DefaultCrawlCacheService.java b/gwtp-core/gwtp-crawler/src/main/java/com/gwtplatform/crawler/server/DefaultCrawlCacheService.java
new file mode 100644
index 0000000000..2bab482cd1
--- /dev/null
+++ b/gwtp-core/gwtp-crawler/src/main/java/com/gwtplatform/crawler/server/DefaultCrawlCacheService.java
@@ -0,0 +1,39 @@
+/*
+ * Copyright 2015 ArcBees Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package com.gwtplatform.crawler.server;
+
+/**
+ * Default crawl cache service implementation.
+ * @author Ben Dol
+ */
+public class DefaultCrawlCacheService implements CrawlCacheService {
+ @Override
+ public DefaultCrawledPage createCrawledPage() {
+ return new DefaultCrawledPage();
+ }
+
+ @Override
+ public DefaultCrawledPage getCachedPage(String url) {
+ return null;
+ }
+
+ @Override
+ public void saveCachedPage(DefaultCrawledPage crawledPage) { }
+
+ @Override
+ public void deleteCachedPage(DefaultCrawledPage crawledPage) { }
+}
diff --git a/gwtp-core/gwtp-crawler/src/main/java/com/gwtplatform/crawler/server/DefaultCrawledPage.java b/gwtp-core/gwtp-crawler/src/main/java/com/gwtplatform/crawler/server/DefaultCrawledPage.java
new file mode 100644
index 0000000000..9eced45418
--- /dev/null
+++ b/gwtp-core/gwtp-crawler/src/main/java/com/gwtplatform/crawler/server/DefaultCrawledPage.java
@@ -0,0 +1,75 @@
+/*
+ * Copyright 2015 ArcBees Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package com.gwtplatform.crawler.server;
+
+import java.util.Date;
+
+/**
+ * Default crawled page implementation.
+ * @author Ben Dol
+ */
+public class DefaultCrawledPage implements CrawledPage {
+ private String url;
+ private Date fetchDate;
+ private boolean fetchInProgress;
+ private String content;
+
+ @Override
+ public void setUrl(String url) {
+ this.url = url;
+ }
+
+ @Override
+ public String getUrl() {
+ return url;
+ }
+
+ @Override
+ public void setFetchDate(Date fetchDate) {
+ this.fetchDate = new Date(fetchDate.getTime());
+ }
+
+ @Override
+ public Date getFetchDate() {
+ return new Date(fetchDate.getTime());
+ }
+
+ @Override
+ public void setFetchInProgress(boolean fetchInProgress) {
+ this.fetchInProgress = fetchInProgress;
+ }
+
+ @Override
+ public boolean isFetchInProgress() {
+ return fetchInProgress;
+ }
+
+ @Override
+ public void setContent(String content) {
+ this.content = content;
+ }
+
+ @Override
+ public String getContent() {
+ return content;
+ }
+
+ @Override
+ public boolean isExpired(long cachedPageTimeoutSec) {
+ return new Date().getTime() > fetchDate.getTime() + cachedPageTimeoutSec * 1000;
+ }
+}
diff --git a/gwtp-crawler-service/src/main/java/com/gwtplatform/crawlerservice/server/InvalidKeyException.java b/gwtp-core/gwtp-crawler/src/main/java/com/gwtplatform/crawler/server/InvalidKeyException.java
similarity index 94%
rename from gwtp-crawler-service/src/main/java/com/gwtplatform/crawlerservice/server/InvalidKeyException.java
rename to gwtp-core/gwtp-crawler/src/main/java/com/gwtplatform/crawler/server/InvalidKeyException.java
index ab5481dd52..37edb10691 100644
--- a/gwtp-crawler-service/src/main/java/com/gwtplatform/crawlerservice/server/InvalidKeyException.java
+++ b/gwtp-core/gwtp-crawler/src/main/java/com/gwtplatform/crawler/server/InvalidKeyException.java
@@ -14,7 +14,7 @@
* the License.
*/
-package com.gwtplatform.crawlerservice.server;
+package com.gwtplatform.crawler.server;
public class InvalidKeyException extends Exception {
private static final long serialVersionUID = 1L;
diff --git a/gwtp-core/pom.xml b/gwtp-core/pom.xml
index 9ebdf5bcce..e857f612d7 100644
--- a/gwtp-core/pom.xml
+++ b/gwtp-core/pom.xml
@@ -29,6 +29,8 @@
gwtp-tester
gwtp-processors
gwtp-crawler
+ gwtp-crawler-guice
+ gwtp-crawler-spring
gwtp-all
@@ -251,6 +253,13 @@
${velocity.version}
+
+
+ net.sourceforge.htmlunit
+ htmlunit
+ ${htmlunit.version}
+
+
org.jukito
diff --git a/gwtp-crawler-service/pom.xml b/gwtp-crawler-service/pom.xml
index 3fc0e2be92..fd5c6dbc36 100644
--- a/gwtp-crawler-service/pom.xml
+++ b/gwtp-crawler-service/pom.xml
@@ -80,6 +80,12 @@
+
+ ${project.groupId}
+ gwtp-crawler-guice
+ ${project.version}
+
+
javax.servlet
servlet-api
@@ -91,12 +97,6 @@
${javax.inject.version}
-
- net.sourceforge.htmlunit
- htmlunit
- ${htmlunit.version}
-
-
com.google.appengine
appengine-api-1.0-sdk
diff --git a/gwtp-crawler-service/src/main/java/com/gwtplatform/crawlerservice/server/OfyCrawlCacheService.java b/gwtp-crawler-service/src/main/java/com/gwtplatform/crawlerservice/server/OfyCrawlCacheService.java
new file mode 100644
index 0000000000..e0c719947e
--- /dev/null
+++ b/gwtp-crawler-service/src/main/java/com/gwtplatform/crawlerservice/server/OfyCrawlCacheService.java
@@ -0,0 +1,60 @@
+/*
+ * Copyright 2015 ArcBees Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package com.gwtplatform.crawlerservice.server;
+
+import javax.inject.Inject;
+import javax.inject.Singleton;
+
+import com.googlecode.objectify.Key;
+import com.gwtplatform.crawler.server.CrawlCacheService;
+import com.gwtplatform.crawlerservice.server.domain.CachedPage;
+import com.gwtplatform.crawlerservice.server.service.CachedPageDao;
+
+/**
+ * Objectify DAO Crawl Cache Service.
+ * @author Ben Dol
+ */
+@Singleton
+public class OfyCrawlCacheService implements CrawlCacheService {
+
+ private final CachedPageDao cachedPageDao;
+
+ @Inject
+ protected OfyCrawlCacheService(CachedPageDao cachedPageDao) {
+ this.cachedPageDao = cachedPageDao;
+ }
+
+ @Override
+ public CachedPage createCrawledPage() {
+ return new CachedPage();
+ }
+
+ @Override
+ public CachedPage getCachedPage(String url) {
+ return cachedPageDao.get(Key.create(CachedPage.class, url));
+ }
+
+ @Override
+ public void saveCachedPage(CachedPage cachedPage) {
+ cachedPageDao.put(cachedPage);
+ }
+
+ @Override
+ public void deleteCachedPage(CachedPage cachedPage) {
+ cachedPageDao.delete(cachedPage);
+ }
+}
diff --git a/gwtp-crawler-service/src/main/java/com/gwtplatform/crawlerservice/server/domain/CachedPage.java b/gwtp-crawler-service/src/main/java/com/gwtplatform/crawlerservice/server/domain/CachedPage.java
index d5ca1cf6e1..65562dc256 100644
--- a/gwtp-crawler-service/src/main/java/com/gwtplatform/crawlerservice/server/domain/CachedPage.java
+++ b/gwtp-crawler-service/src/main/java/com/gwtplatform/crawlerservice/server/domain/CachedPage.java
@@ -20,50 +20,60 @@
import com.googlecode.objectify.annotation.Entity;
import com.googlecode.objectify.annotation.Id;
+import com.gwtplatform.crawler.server.CrawledPage;
/**
* Stores a cached version of a page.
*/
@Entity
-public class CachedPage {
+public class CachedPage implements CrawledPage {
@Id
private String url;
private Date fetchDate;
private boolean fetchInProgress;
private String content;
+ @Override
public void setUrl(String url) {
this.url = url;
}
+ @Override
public String getUrl() {
return url;
}
+ @Override
public void setFetchDate(Date fetchDate) {
this.fetchDate = new Date(fetchDate.getTime());
}
+ @Override
public Date getFetchDate() {
return new Date(fetchDate.getTime());
}
+ @Override
public void setFetchInProgress(boolean fetchInProgress) {
this.fetchInProgress = fetchInProgress;
}
+ @Override
public boolean isFetchInProgress() {
return fetchInProgress;
}
+ @Override
public void setContent(String content) {
this.content = content;
}
+ @Override
public String getContent() {
return content;
}
+ @Override
public boolean isExpired(long cachedPageTimeoutSec) {
return new Date().getTime() > fetchDate.getTime() + cachedPageTimeoutSec * 1000;
}
diff --git a/gwtp-crawler-service/src/main/java/com/gwtplatform/crawlerservice/server/guice/CrawlServiceModule.java b/gwtp-crawler-service/src/main/java/com/gwtplatform/crawlerservice/server/guice/CrawlServiceModule.java
index a6c4f0f6e1..b3dc04f27c 100644
--- a/gwtp-crawler-service/src/main/java/com/gwtplatform/crawlerservice/server/guice/CrawlServiceModule.java
+++ b/gwtp-crawler-service/src/main/java/com/gwtplatform/crawlerservice/server/guice/CrawlServiceModule.java
@@ -22,12 +22,16 @@
import com.gargoylesoftware.htmlunit.WebClient;
import com.google.inject.Provides;
import com.google.inject.servlet.ServletModule;
-import com.gwtplatform.crawlerservice.server.CrawlServiceServlet;
+import com.gwtplatform.crawler.server.CrawlCacheService;
+import com.gwtplatform.crawler.server.guice.service.CrawlServiceServlet;
+import com.gwtplatform.crawlerservice.server.OfyCrawlCacheService;
public class CrawlServiceModule extends ServletModule {
@Override
public void configureServlets() {
+ bind(CrawlCacheService.class).to(OfyCrawlCacheService.class);
+
serve("*").with(CrawlServiceServlet.class);
}