Skip to content
Open
Show file tree
Hide file tree
Changes from 24 commits
Commits
Show all changes
36 commits
Select commit Hold shift + click to select a range
c8a80a7
Finished Testing ElasticSearch
Mar 31, 2017
acb9a99
Fixed merge conflicts
Mar 31, 2017
c1b319f
Fixed incorrect overriding of superclass methods
Mar 31, 2017
720ffdf
Changed tests to make sure that we pass in a Tag into TagFactory clas…
Mar 31, 2017
ac25c16
Removed delete request
Mar 31, 2017
47257da
New data
Mar 31, 2017
a87ea7f
removed unnecessary data files
Mar 31, 2017
03b2f8e
Unnecessary pom.xml file
Mar 31, 2017
553d079
moved pom.xml to correct location
Mar 31, 2017
ef72d5d
Revert "removed unnecessary data files"
Mar 31, 2017
2e709e5
Revert "New data"
Mar 31, 2017
3bc5466
removed unnecessary data files
Mar 31, 2017
73c86d9
added ElasticSearch dependencies
Mar 31, 2017
f22645c
added line to insert into ElasticSearch
Mar 31, 2017
8ffb0fe
changed parameter back to string -- more intuitive interface for TagR…
Mar 31, 2017
357b429
modified method signature for inserting into DB to throw GroundExcept…
Mar 31, 2017
450bcf0
Cleaned up file, removed main method that was used for testing purposes
Mar 31, 2017
77511ce
Created TagsResource API
Mar 31, 2017
aa50b59
added method to FactoryGenerator interface to add TagsResource
Mar 31, 2017
0e1b747
added tags resource
Mar 31, 2017
1dac8ec
Fixed connection to elasticsearch
Apr 2, 2017
e788608
Commiting changes so I can pull from upstream:
Apr 8, 2017
1253014
Resolved merge conflicts and added teardown methods to Neo4jTest.java…
Apr 8, 2017
b150228
Fixed Mapper Parsing Errors
Apr 10, 2017
a639cc0
committing for the purpose of pulling from upstream
Apr 18, 2017
a6b25a1
Fixed merge conflicts
Apr 18, 2017
cb6dcc0
fixed minor bugs
Apr 18, 2017
2f33f01
added ElasticSearch version in correct format
Apr 18, 2017
2e79918
Fixed issue with *name
Apr 18, 2017
bc7dc23
added elasticsearch version property
Apr 18, 2017
a881dc1
Modified so that refresh indices occurs only on reads, not writes
Apr 18, 2017
648c970
Fixed whitespace issues and handles exceptions more robustly
Apr 18, 2017
fd1c51f
added exception class for ElasticSearch
Apr 18, 2017
39656a3
Committing for the purpose of pulling from upstream
Apr 24, 2017
4d7473c
commiting to pull from upstream
May 2, 2017
7b10afb
reformatted code to deal with elasticSearch option
May 2, 2017
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,11 @@
<artifactId>jline</artifactId>
<version>2.12</version>
</dependency>
<dependency>
<groupId>org.elasticsearch</groupId>
<artifactId>elasticsearch</artifactId>
<version>1.5.1</version>

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The version should be a variable in the properties field of the POM.

</dependency>

</dependencies>

Expand Down
20 changes: 9 additions & 11 deletions src/main/java/edu/berkeley/ground/GroundServer.java
Original file line number Diff line number Diff line change
Expand Up @@ -14,14 +14,7 @@

package edu.berkeley.ground;

import edu.berkeley.ground.dao.models.EdgeFactory;
import edu.berkeley.ground.dao.models.EdgeVersionFactory;
import edu.berkeley.ground.dao.models.GraphFactory;
import edu.berkeley.ground.dao.models.GraphVersionFactory;
import edu.berkeley.ground.dao.models.NodeFactory;
import edu.berkeley.ground.dao.models.NodeVersionFactory;
import edu.berkeley.ground.dao.models.StructureFactory;
import edu.berkeley.ground.dao.models.StructureVersionFactory;
import edu.berkeley.ground.dao.models.*;

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We're using the Google Java style guide which specifies no * imports.

import edu.berkeley.ground.dao.usage.LineageEdgeFactory;
import edu.berkeley.ground.dao.usage.LineageEdgeVersionFactory;
import edu.berkeley.ground.dao.usage.LineageGraphFactory;
Expand All @@ -37,6 +30,7 @@
import edu.berkeley.ground.resources.LineageGraphsResource;
import edu.berkeley.ground.resources.NodesResource;
import edu.berkeley.ground.resources.StructuresResource;
import edu.berkeley.ground.resources.TagsResource;
import edu.berkeley.ground.util.CassandraFactories;
import edu.berkeley.ground.util.FactoryGenerator;
import edu.berkeley.ground.util.Neo4jFactories;
Expand All @@ -60,9 +54,11 @@ public class GroundServer extends Application<GroundServerConfiguration> {
private StructureVersionFactory structureVersionFactory;
private LineageGraphFactory lineageGraphFactory;
private LineageGraphVersionFactory lineageGraphVersionFactory;
private TagFactory tagFactory;

public static void main(String[] args) throws Exception {
new GroundServer().run(args);

}

@Override
Expand Down Expand Up @@ -135,16 +131,17 @@ public void run(GroundServerConfiguration configuration, Environment environment
this.structureVersionFactory,
dbClient);
final LineageGraphsResource lineageGraphsResource = new LineageGraphsResource(
this.lineageGraphFactory,
this.lineageGraphVersionFactory,
dbClient);
this.lineageGraphFactory, this.lineageGraphVersionFactory, dbClient);
final TagsResource tagsResource = new TagsResource(this.tagFactory);


environment.jersey().register(edgesResource);
environment.jersey().register(graphsResource);
environment.jersey().register(lineageEdgesResource);
environment.jersey().register(nodesResource);
environment.jersey().register(structuresResource);
environment.jersey().register(lineageGraphsResource);
environment.jersey().register(tagsResource);
}

private void setPostgresFactories(PostgresClient postgresClient, int machineId, int numMachines) {
Expand Down Expand Up @@ -174,5 +171,6 @@ private void setFactories(FactoryGenerator factoryGenerator) {
structureVersionFactory = factoryGenerator.getStructureVersionFactory();
lineageGraphFactory = factoryGenerator.getLineageGraphFactory();
lineageGraphVersionFactory = factoryGenerator.getLineageGraphVersionFactory();
tagFactory = factoryGenerator.getTagFactory();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -28,4 +28,5 @@ public abstract class TagFactory {
public abstract List<Long> getVersionIdsByTag(String tag) throws GroundException;

public abstract List<Long> getItemIdsByTag(String tag) throws GroundException;

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please remove unnecessary whitespace.

}
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
* limitations under the License.
*/


package edu.berkeley.ground.dao.models.cassandra;

import edu.berkeley.ground.dao.models.RichVersionFactory;
Expand All @@ -26,6 +27,7 @@
import edu.berkeley.ground.model.models.StructureVersion;
import edu.berkeley.ground.model.models.Tag;
import edu.berkeley.ground.model.versions.GroundType;
import edu.berkeley.ground.util.ElasticSearch;

import java.util.ArrayList;
import java.util.HashMap;
Expand Down Expand Up @@ -91,7 +93,7 @@ public void insertIntoDatabase(long id,

for (String key : tags.keySet()) {
Tag tag = tags.get(key);

ElasticSearch.insertElasticSearch(tag, "rich_version");
List<DbDataContainer> tagInsertion = new ArrayList<>();
tagInsertion.add(new DbDataContainer("rich_version_id", GroundType.LONG, id));
tagInsertion.add(new DbDataContainer("key", GroundType.STRING, key));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@

package edu.berkeley.ground.dao.models.cassandra;


import edu.berkeley.ground.dao.models.TagFactory;
import edu.berkeley.ground.db.CassandraClient;
import edu.berkeley.ground.db.CassandraResults;
Expand All @@ -23,6 +24,7 @@
import edu.berkeley.ground.exceptions.GroundException;
import edu.berkeley.ground.model.models.Tag;
import edu.berkeley.ground.model.versions.GroundType;
import edu.berkeley.ground.util.ElasticSearch;

import java.util.ArrayList;
import java.util.HashMap;
Expand Down Expand Up @@ -76,16 +78,17 @@ private Map<String, Tag> retrieveFromDatabaseById(long id, String keyPrefix)
return result;
}

@Override
public List<Long> getVersionIdsByTag(String tag) throws GroundException {
return this.getIdsByTag(tag, "rich_version");
return ElasticSearch.getSearchResponse("rich_version", tag);

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We should probably have a configuration option that allows people to turn on / off ElasticSearch use. Let's add a field to the config and pass it through to the TagFactory. If it's turned off, we can go to the regular database (current impl.), and if it's turned on, we can use the ElasticSearch API.

}

@Override
public List<Long> getItemIdsByTag(String tag) throws GroundException {
return this.getIdsByTag(tag, "item");
return ElasticSearch.getSearchResponse("item", tag);

}



private List<Long> getIdsByTag(String tag, String keyPrefix) throws GroundException {
List<Long> result = new ArrayList<>();

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@

package edu.berkeley.ground.dao.models.neo4j;


import edu.berkeley.ground.dao.models.RichVersionFactory;
import edu.berkeley.ground.db.DbDataContainer;
import edu.berkeley.ground.db.Neo4jClient;
Expand All @@ -30,6 +31,7 @@
import java.util.List;
import java.util.Map;

import edu.berkeley.ground.util.ElasticSearch;
import org.neo4j.driver.internal.value.NullValue;
import org.neo4j.driver.internal.value.StringValue;
import org.neo4j.driver.v1.Record;
Expand Down Expand Up @@ -101,7 +103,7 @@ public void insertIntoDatabase(long id,

for (String key : tags.keySet()) {
Tag tag = tags.get(key);

ElasticSearch.insertElasticSearch(tag, "rich_version");
List<DbDataContainer> tagInsertion = new ArrayList<>();
tagInsertion.add(new DbDataContainer("rich_version_id", GroundType.LONG, id));
tagInsertion.add(new DbDataContainer("tkey", GroundType.STRING, key));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import java.util.List;
import java.util.Map;

import edu.berkeley.ground.util.ElasticSearch;
import org.neo4j.driver.internal.value.NullValue;
import org.neo4j.driver.internal.value.StringValue;
import org.neo4j.driver.v1.Record;
Expand Down Expand Up @@ -86,14 +87,12 @@ private Map<String, Tag> retrieveFromDatabaseById(long id, String keyPrefix)
return tags;
}

@Override
public List<Long> getVersionIdsByTag(String tag) throws GroundDbException {
return this.getIdsByTag(tag, "rich_version_id");
public List<Long> getVersionIdsByTag(String tag) throws GroundException {
return ElasticSearch.getSearchResponse("rich_version", tag);
}

@Override
public List<Long> getItemIdsByTag(String tag) throws GroundDbException {
return this.getIdsByTag(tag, "item_id");
public List<Long> getItemIdsByTag(String tag) throws GroundException {
return ElasticSearch.getSearchResponse("item", tag);
}

private List<Long> getIdsByTag(String tag, String idAttribute) throws GroundDbException {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@

package edu.berkeley.ground.dao.models.postgres;


import edu.berkeley.ground.dao.models.RichVersionFactory;
import edu.berkeley.ground.dao.versions.postgres.PostgresVersionFactory;
import edu.berkeley.ground.db.DbClient;
Expand All @@ -26,6 +27,7 @@
import edu.berkeley.ground.model.models.StructureVersion;
import edu.berkeley.ground.model.models.Tag;
import edu.berkeley.ground.model.versions.GroundType;
import edu.berkeley.ground.util.ElasticSearch;

import java.util.ArrayList;
import java.util.HashMap;
Expand Down Expand Up @@ -91,7 +93,7 @@ public void insertIntoDatabase(long id,

for (String key : tags.keySet()) {
Tag tag = tags.get(key);

ElasticSearch.insertElasticSearch(tag, "rich_version");
List<DbDataContainer> tagInsertion = new ArrayList<>();
tagInsertion.add(new DbDataContainer("rich_version_id", GroundType.LONG, id));
tagInsertion.add(new DbDataContainer("key", GroundType.STRING, key));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import edu.berkeley.ground.exceptions.GroundException;
import edu.berkeley.ground.model.models.Tag;
import edu.berkeley.ground.model.versions.GroundType;
import edu.berkeley.ground.util.ElasticSearch;

import java.util.ArrayList;
import java.util.HashMap;
Expand Down Expand Up @@ -77,13 +78,13 @@ private Map<String, Tag> retrieveFromDatabaseById(long id, String keyPrefix)

@Override
public List<Long> getVersionIdsByTag(String tag) throws GroundException {
return this.getIdsByTag(tag, "rich_version");
return ElasticSearch.getSearchResponse("rich_version", tag);
}


@Override
public List<Long> getItemIdsByTag(String tag) throws GroundException {
return this.getIdsByTag(tag, "item");
return ElasticSearch.getSearchResponse("item", tag);
}

private List<Long> getIdsByTag(String tag, String keyPrefix) throws GroundException {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import java.util.List;
import java.util.Map;

import edu.berkeley.ground.util.ElasticSearch;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand Down Expand Up @@ -69,7 +70,7 @@ public void insertIntoDatabase(long id, Map<String, Tag> tags) throws GroundExce

for (String key : tags.keySet()) {
Tag tag = tags.get(key);

ElasticSearch.insertElasticSearch(tag, "item");
List<DbDataContainer> tagInsertion = new ArrayList<>();
tagInsertion.add(new DbDataContainer("item_id", GroundType.LONG, id));
tagInsertion.add(new DbDataContainer("key", GroundType.STRING, key));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,6 @@ public CassandraVersionFactory(CassandraClient dbClient) {
public void insertIntoDatabase(long id) throws GroundException {
List<DbDataContainer> insertions = new ArrayList<>();
insertions.add(new DbDataContainer("id", GroundType.LONG, id));

this.dbClient.insert("version", insertions);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import edu.berkeley.ground.model.versions.GroundType;
import edu.berkeley.ground.model.versions.Item;
import edu.berkeley.ground.model.versions.VersionHistoryDag;
import edu.berkeley.ground.util.ElasticSearch;

import java.util.ArrayList;
import java.util.List;
Expand Down Expand Up @@ -57,9 +58,10 @@ public Neo4jItemFactory(Neo4jClient dbClient,
* @throws GroundDbException an error inserting data into the database
*/
@Override
public void insertIntoDatabase(long id, Map<String, Tag> tags) throws GroundDbException {
public void insertIntoDatabase(long id, Map<String, Tag> tags) throws GroundException {
for (String key : tags.keySet()) {
Tag tag = tags.get(key);
ElasticSearch.insertElasticSearch(tag, "item");

List<DbDataContainer> tagInsertion = new ArrayList<>();
tagInsertion.add(new DbDataContainer("item_id", GroundType.LONG, id));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import java.util.List;
import java.util.Map;

import edu.berkeley.ground.util.ElasticSearch;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Expand Down Expand Up @@ -69,6 +70,7 @@ public void insertIntoDatabase(long id, Map<String, Tag> tags) throws GroundExce

for (String key : tags.keySet()) {
Tag tag = tags.get(key);
ElasticSearch.insertElasticSearch(tag, "item");

List<DbDataContainer> tagInsertion = new ArrayList<>();
tagInsertion.add(new DbDataContainer("item_id", GroundType.LONG, id));
Expand Down
65 changes: 65 additions & 0 deletions src/main/java/edu/berkeley/ground/resources/TagsResource.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
package edu.berkeley.ground.resources;

import com.codahale.metrics.annotation.Timed;
import edu.berkeley.ground.dao.models.TagFactory;
import edu.berkeley.ground.exceptions.GroundException;
import edu.berkeley.ground.model.models.Tag;
import io.swagger.annotations.Api;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import javax.ws.rs.*;
import javax.ws.rs.core.MediaType;
import java.util.List;
import java.util.Map;


@Path("/tags")
@Api(value = "/tags", description = "Interact with tags")
@Produces(MediaType.APPLICATION_JSON)
@Consumes(MediaType.APPLICATION_JSON)
public class TagsResource {
private static final Logger LOGGER = LoggerFactory.getLogger(NodesResource.class);

private final TagFactory tagFactory;
public TagsResource(TagFactory tagFactory) {
this.tagFactory = tagFactory;

}

@GET
@Timed
@Path("/versions/{id}")
public Map<String, Tag> retrieveFromDatabaseByVersionId(@PathParam("id") long id) throws GroundException {
LOGGER.info("Retrieving version with id: " + id + ".");
return this.tagFactory.retrieveFromDatabaseByVersionId(id);
}

@GET
@Timed
@Path("/items/{id}")
public Map<String, Tag> retrieveFromDatabaseByItemId(@PathParam("id") long id) throws GroundException {
LOGGER.info("Retrieving item with id: " + id + ".");
return this.tagFactory.retrieveFromDatabaseByItemId(id);
}

@GET
@Timed
@Path("/versions/{*name}")

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is there a reason there's a * in the variable name?

public List<Long> getVersionIdsByTag(@PathParam("*name") String tag) throws GroundException {
LOGGER.info("Retrieving all version ids with tag: " + tag + ".");
return this.tagFactory.getVersionIdsByTag(tag);
}

@GET
@Timed
@Path("/items/{*name}")
public List<Long> getItemIdsByTag(@PathParam("*name") String tag) throws GroundException {
LOGGER.info("Retrieving all item ids with tag: " + tag + ".");
return this.tagFactory.getItemIdsByTag(tag);
}

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Whitespace.




}
Loading