litvinovg/vitro
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'issue-vivo-98' into develop

Browse source
This commit is contained in:
Brian Caruso 2013-07-16 12:46:05 -04:00
commit 2c146f9a86
8 changed files with 528 additions and 20 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
webapp/src/edu/cornell/mannlib/vitro/webapp/filestorage/uploadrequest/FileUploadServletRequest.java
View file

@ -2,21 +2,12 @@
package edu.cornell.mannlib.vitro.webapp.filestorage.uploadrequest;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.security.Principal;
import java.util.Enumeration;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import javax.servlet.RequestDispatcher;
import javax.servlet.ServletInputStream;
import javax.servlet.http.Cookie;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletRequestWrapper;
import javax.servlet.http.HttpSession;
import org.apache.commons.fileupload.FileItem;
import org.apache.commons.fileupload.FileUploadException;

61
webapp/src/edu/cornell/mannlib/vitro/webapp/filestorage/uploadrequest/MultipartHttpServletRequest.java
View file

@ -29,29 +29,60 @@ import org.apache.commons.logging.LogFactory;
* any parameter-related requests. File-related information will also be held
* here, to answer file-related requests.
*/
class MultipartHttpServletRequest extends FileUploadServletRequest {
public class MultipartHttpServletRequest extends FileUploadServletRequest {
private static final Log log = LogFactory
.getLog(MultipartHttpServletRequest.class);
private static final String[] EMPTY_ARRAY = new String[0];
private final Map<String, List<String>> parameters;
private final Map<String, List<FileItem>> files;
private Map<String, List<String>> parameters;
private Map<String, List<FileItem>> files;
private FileUploadException fileUploadException;
private boolean storeFilesToTempDir = false;
private int maxFileSize = 0;
private File tempDir = null;
/**
* Parse the multipart request. Store the info about the request parameters.
* Don't store the uploaded files to a temporary directory to allow streaming.
*
* Only use this constructor if you plan to consume the FileItems using streaming
* to deal with inputs of very large sizes.
*
* In all other case you should use the maxFileSize constructor to deal with
* the size of the uploaded file in a safe way.
*/
public MultipartHttpServletRequest(HttpServletRequest request)
throws IOException{
super(request);
storeFilesToTempDir = false;
}
/**
* Parse the multipart request. Store the info about the request parameters
* and the uploaded files.
* and the uploaded files to a temporary directory.
*
* This offers a simple way to deal with uploaded files by having a size
* limit. This limit may be rather large if desired. Many megabytes for example.
*/
public MultipartHttpServletRequest(HttpServletRequest request,
int maxFileSize) throws IOException {
super(request);
storeFilesToTempDir = true;
this.maxFileSize = maxFileSize;
this.tempDir = figureTemporaryDirectory(request);
}
private void setup(HttpServletRequest request){
Map<String, List<String>> parameters = new HashMap<String, List<String>>();
Map<String, List<FileItem>> files = new HashMap<String, List<FileItem>>();
File tempDir = figureTemporaryDirectory(request);
ServletFileUpload upload = createUploadHandler(maxFileSize, tempDir);
ServletFileUpload upload = createUploadHandler();
//File tempDir = figureTemporaryDirectory(request);
//ServletFileUpload upload = createUploadHandler(maxFileSize, tempDir);
parseQueryString(request.getQueryString(), parameters);
@ -75,14 +106,16 @@ class MultipartHttpServletRequest extends FileUploadServletRequest {
fileUploadException = e;
request.setAttribute(
FileUploadServletRequest.FILE_UPLOAD_EXCEPTION, e);
}
} catch (UnsupportedEncodingException e) {
log.error("could not convert to UTF-8",e);
}
this.parameters = Collections.unmodifiableMap(parameters);
log.debug("Parameters are: " + this.parameters);
this.files = Collections.unmodifiableMap(files);
log.debug("Files are: " + this.files);
request.setAttribute(FILE_ITEM_MAP, this.files);
}
}
/**
* Pull any parameters out of the URL.
@ -131,6 +164,18 @@ class MultipartHttpServletRequest extends FileUploadServletRequest {
"javax.servlet.context.tempdir");
}
/**
* Create an upload handler based on this.storeFilesToTempDir.
*/
private ServletFileUpload createUploadHandler(){
if( storeFilesToTempDir ){
return createUploadHandler( this.maxFileSize, this.tempDir );
}else{
return new ServletFileUpload();
}
}
/**
* Create an upload handler that will throw an exception if the file is too
* large.

153
webapp/src/edu/cornell/mannlib/vitro/webapp/search/controller/SearchServiceController.java Normal file
View file

@ -0,0 +1,153 @@
/* $This file is distributed under the terms of the license in /doc/license.txt$ */
package edu.cornell.mannlib.vitro.webapp.search.controller;
import java.io.IOException;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import edu.cornell.mannlib.vitro.webapp.auth.identifier.ActiveIdentifierBundleFactories;
import edu.cornell.mannlib.vitro.webapp.auth.identifier.IdentifierBundle;
import edu.cornell.mannlib.vitro.webapp.auth.permissions.SimplePermission;
import edu.cornell.mannlib.vitro.webapp.auth.policy.PolicyHelper;
import edu.cornell.mannlib.vitro.webapp.auth.policy.ServletPolicyList;
import edu.cornell.mannlib.vitro.webapp.auth.policy.ifaces.PolicyIface;
import edu.cornell.mannlib.vitro.webapp.auth.requestedAction.Actions;
import edu.cornell.mannlib.vitro.webapp.beans.UserAccount;
import edu.cornell.mannlib.vitro.webapp.controller.VitroRequest;
import edu.cornell.mannlib.vitro.webapp.controller.authenticate.Authenticator;
import edu.cornell.mannlib.vitro.webapp.controller.authenticate.BasicAuthenticator;
import edu.cornell.mannlib.vitro.webapp.controller.freemarker.FreemarkerHttpServlet;
import edu.cornell.mannlib.vitro.webapp.controller.freemarker.responsevalues.ExceptionResponseValues;
import edu.cornell.mannlib.vitro.webapp.controller.freemarker.responsevalues.ResponseValues;
import edu.cornell.mannlib.vitro.webapp.controller.freemarker.responsevalues.TemplateResponseValues;
import edu.cornell.mannlib.vitro.webapp.filestorage.uploadrequest.MultipartHttpServletRequest;
import edu.cornell.mannlib.vitro.webapp.search.indexing.IndexBuilder;
/**
* Accepts requests to update a set of URIs in the search index.
*/
@SuppressWarnings("serial")
public class SearchServiceController extends FreemarkerHttpServlet {
private static final Log log = LogFactory.getLog(SearchServiceController.class);
/**
* Attempt to check if there is a email and password in the request parameters.
* If there is not, fall back on the normal usage pattern of requestedAction().
* If there is, then try to authenticate and authorize the
* userAccount associated with the email.
*/
@Override
protected Actions requiredActions(VitroRequest vreq) {
try{
// Works by side effect: parse the multi-part request and stash FileItems in request
new MultipartHttpServletRequest( vreq );
//first figure out if the password and email login to an account with a password
String pw = vreq.getParameter("password");
String email = vreq.getParameter("email");
log.debug(String.format("email: '%s' password: '%s' ",email,pw));
if( pw == null || email == null || pw.isEmpty() || email.isEmpty()){
return SimplePermission.MANAGE_SEARCH_INDEX.ACTIONS;
}
Authenticator basicAuth = new BasicAuthenticator(vreq);
UserAccount user = basicAuth.getAccountForInternalAuth( email );
log.debug("userAccount is " + user==null?"null":user.getUri() );
if( ! basicAuth.isCurrentPassword( user, pw ) ){
log.debug(String.format("UNAUTHORIZED, password not accepted for %s, account URI: %s",
user.getEmailAddress(), user.getUri()));
return Actions.UNAUTHORIZED;
}else{
log.debug(String.format("password accepted for %s, account URI: %s",
user.getEmailAddress(), user.getUri() ));
}
//then figure out if that account can manage the search index.
IdentifierBundle ids =
ActiveIdentifierBundleFactories.getUserIdentifierBundle(vreq,user);
PolicyIface policy = ServletPolicyList.getPolicies(vreq);
boolean canManageSearchIndex =
PolicyHelper.isAuthorizedForActions( ids, policy,
SimplePermission.MANAGE_SEARCH_INDEX.ACTIONS );
if( canManageSearchIndex ){
return Actions.AUTHORIZED;
}else{
log.debug(String.format("userAccount is unauthorized to" +
" manage the search index.",user.getUri()));
return Actions.UNAUTHORIZED;
}
}catch(Exception ex){
log.error("Error while attempting to log in " +
"to SearchServiceController: " + ex.getMessage());
return Actions.UNAUTHORIZED;
}
}
/**
* Handle the different actions. If not specified, the default action is to
* show the help page.
*/
@Override
protected ResponseValues processRequest(VitroRequest req) {
try {
//figure out what action to perform
String pathInfo = req.getPathInfo();
if( pathInfo == null || pathInfo.trim().isEmpty() || "/".equals(pathInfo.trim()) ){
return doHelpForm(req);
}
pathInfo = pathInfo.substring(1); //get rid of leading slash
if (VERBS.UPDATE_URIS_IN_SEARCH.verb.equals( pathInfo )) {
return doUpdateUrisInSearch(req);
} else {
return doHelpForm(req);
}
} catch (Exception e) {
return new ExceptionResponseValues(e);
}
}
/**
* Process requests to the web service to update a list of URIs in the search index. */
public ResponseValues doUpdateUrisInSearch(HttpServletRequest req )
throws IOException, ServletException {
IndexBuilder builder = IndexBuilder.getBuilder(getServletContext());
if( builder == null )
throw new ServletException( "Could not get search index builder from context. Check smoke test");
new UpdateUrisInIndex().doUpdateUris( req, builder);
TemplateResponseValues trv = new TemplateResponseValues( "" );
return trv;
}
public ResponseValues doHelpForm(HttpServletRequest req){
return new TemplateResponseValues( "searchService-help.ftl");
}
public enum VERBS{
UPDATE_URIS_IN_SEARCH("updateUrisInSearch");
public final String verb;
VERBS(String verb){
this.verb = verb;
}
}
}

192
webapp/src/edu/cornell/mannlib/vitro/webapp/search/controller/UpdateUrisInIndex.java Normal file
View file

@ -0,0 +1,192 @@
/* $This file is distributed under the terms of the license in /doc/license.txt$ */
package edu.cornell.mannlib.vitro.webapp.search.controller;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import java.util.regex.Pattern;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.apache.commons.fileupload.FileItemIterator;
import org.apache.commons.fileupload.FileItemStream;
import org.apache.commons.fileupload.FileUploadException;
import org.apache.commons.fileupload.servlet.ServletFileUpload;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import edu.cornell.mannlib.vitro.webapp.search.indexing.IndexBuilder;
/**
* Class that performs the update of the uris in the search index
* for the SearchService.
*/
public class UpdateUrisInIndex {
private static final Log log = LogFactory.getLog(UpdateUrisInIndex.class);
/**
* Web service for update in search index of a list of URIs.
* @throws IOException
*/
protected void doUpdateUris(HttpServletRequest req, IndexBuilder builder)
throws ServletException, IOException{
if( ! ServletFileUpload.isMultipartContent(req) )
throw new ServletException("Expected Multipart Content");
String enc = getEncoding(req);
try{
//loop over the fileds and add any URIs to the IndexBuilder queue
ServletFileUpload upload = new ServletFileUpload();
FileItemIterator iter = upload.getItemIterator(req);
while( iter.hasNext()){
FileItemStream item = iter.next();
String name = item.getFieldName();
if( "email".equals(name) || "password".equals(name) )
continue; //skip the password and email fields
InputStream stream = item.openStream();
try{
addToSearchQueue(builder, stream, enc);
}finally{
stream.close();
}
}
}catch (FileUploadException fex){
throw new ServletException("Could not upload file to SearchServiceController", fex);
}finally{
builder.doUpdateIndex();
}
}
/**
* Get the encoding of the request, default to UTF-8
* since that is in the vitro install instructions
* to put on the connector.
*/
private String getEncoding(HttpServletRequest req){
String enc = req.getCharacterEncoding();
if( enc == null || enc.isEmpty() ){
log.debug("No encoding on POST request, That is acceptable.");
enc = "UTF-8";
}else if( enc.length() > 30){
log.debug("Ignoring odd encoding of '" + enc + "'");
enc = "UTF-8";
}else{
log.debug("Encoding set on POST request: " + enc);
}
log.debug("Reading POSTed URIs with encoding " + enc);
return enc;
}
/**
* Adds URIs from Reader to search queue.
*/
private void addToSearchQueue( IndexBuilder builder, InputStream stream , String charEncoding )
throws IOException{
Iterator<String> uris =
new UrisFromInputIterator( new InputStreamReader(stream, charEncoding) );
while(uris.hasNext()){
String uri = uris.next();
log.debug("Request to index uri '" + uri + "'");
builder.addToChanged( uri );
}
}
/**
* Iterator for URIs in a reader to make top level methods simpler.
*/
public static class UrisFromInputIterator implements Iterator<String> {
BufferedReader reader;
Iterator<String> uris;
public UrisFromInputIterator(Reader in ){
this.reader = new BufferedReader(in);
}
public void remove(){ throw new UnsupportedOperationException() ; }
public boolean hasNext(){
if( uris != null && uris.hasNext() ){
return true;
}else{
try {
return getFromBuffer();
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
return false;
}
}
}
public String next(){
return uris.next();
}
/** Returns true if there are uris to get.
* @throws IOException */
private boolean getFromBuffer() throws IOException{
uris = null;
while( uris == null || !uris.hasNext() ){
String chunk = reader.readLine();
if( chunk == null ){ //at end of input
break;
} else if( chunk.trim().isEmpty() ){
continue;
}else{
uris = lineToUris(chunk).iterator();
if( uris.hasNext() ){
return true;
}
}
}
return false;
}
}
/**
* Removes null and empty elements from in.
* Returned list will not be null.
*/
private static List<String> removeNullAndEmpty(List<String> in ){
ArrayList<String> out = new ArrayList<String>();
if( in == null )
return out;
for( String s : in ){
if( s != null && !s.trim().isEmpty() ){
out.add(s);
}
}
return out;
}
/**
* Parses a line to a list of URIs.
* Retruned list will not be null.
* No elements in returned list will be empty or null.
*/
protected static List<String> lineToUris(String line){
List<String> parts = removeNullAndEmpty( Arrays.asList(commaAndWhitespace.split( line ) ));
return parts;
}
/** Pattern to split URIs on whitespace and commas. */
private static final Pattern commaAndWhitespace = Pattern.compile("[,\\s]");
}

18
webapp/src/edu/cornell/mannlib/vitro/webapp/search/indexing/IndexBuilder.java
View file

@ -20,6 +20,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import com.hp.hpl.jena.query.QueryParseException;
import com.hp.hpl.jena.rdf.model.ResourceFactory;
import com.hp.hpl.jena.rdf.model.Statement;
import edu.cornell.mannlib.vitro.webapp.beans.Individual;
@ -130,12 +131,22 @@ public class IndexBuilder extends VitroBackgroundThread {
* your changes with a call to doUpdateIndex().
*/
public void addToChanged(Statement stmt) {
log.debug("call to addToChanged()");
log.debug("call to addToChanged(Statement)");
synchronized(changedStmts){
changedStmts.add(stmt);
}
}
/**
* Convenience method to add a URI to the change queue.
*/
public void addToChanged(String uri){
addToChanged(ResourceFactory.createStatement(
ResourceFactory.createResource(uri),
ResourceFactory.createProperty("http://ex.com/f"),
ResourceFactory.createPlainLiteral("added by IndexBuilder.addToChanged(uri)")));
}
/**
* This method will cause the IndexBuilder to completely rebuild
* the index.
@ -244,7 +255,8 @@ public class IndexBuilder extends VitroBackgroundThread {
for( StatementToURIsToUpdate stu : stmtToURIsToIndexFunctions ) {
stu.startIndexing();
}
//keep uris unique by using a HashSet
Collection<String> urisToUpdate = new HashSet<String>();
for( Statement stmt : getAndClearChangedStmts() ){
for( StatementToURIsToUpdate stu : stmtToURIsToIndexFunctions ){
@ -256,7 +268,7 @@ public class IndexBuilder extends VitroBackgroundThread {
for( StatementToURIsToUpdate stu : stmtToURIsToIndexFunctions ) {
stu.endIndxing();
}
return urisToUpdate;
}

66
webapp/test/edu/cornell/mannlib/vitro/webapp/search/controller/UpdateUrisInIndexTest.java Normal file
View file

@ -0,0 +1,66 @@
/* $This file is distributed under the terms of the license in /doc/license.txt$ */
package edu.cornell.mannlib.vitro.webapp.search.controller;
import java.io.StringReader;
import java.util.Arrays;
import java.util.Collections;
import java.util.Iterator;
import org.junit.Assert;
import org.junit.Test;
/**
* Accepts requests to update a set of URIs in the search index.
*/
public class UpdateUrisInIndexTest {
@Test
public void lineToUrisTest(){
Assert.assertEquals(Arrays.asList("uri1"), UpdateUrisInIndex.lineToUris( "uri1"));
Assert.assertEquals(Arrays.asList("uri1", "uri2"), UpdateUrisInIndex.lineToUris( "uri1,uri2"));
Assert.assertEquals(Arrays.asList("uri1"), UpdateUrisInIndex.lineToUris( "uri1\n"));
Assert.assertEquals(Arrays.asList("uri1","uri2"), UpdateUrisInIndex.lineToUris( "uri1\nuri2"));
Assert.assertEquals(Collections.EMPTY_LIST, UpdateUrisInIndex.lineToUris( "" ));
Assert.assertEquals(Collections.EMPTY_LIST, UpdateUrisInIndex.lineToUris( "," ));
Assert.assertEquals(Collections.EMPTY_LIST, UpdateUrisInIndex.lineToUris( " , " ));
}
@Test
public void UrisFromInputIteratorTest(){
doUrisFromInputIterator("",0);
doUrisFromInputIterator(" ",0);
doUrisFromInputIterator(" , ",0);
doUrisFromInputIterator("\n",0);
doUrisFromInputIterator("\n\n\n",0);
doUrisFromInputIterator("http://bogus.com/n234",1);
doUrisFromInputIterator("http://bogus.com/n234\nhttp://bogus.com/n442",2);
doUrisFromInputIterator("http://bogus.com/n234, http://bogus.com/n442",2);
doUrisFromInputIterator("http://bogus.com/n234,\nhttp://bogus.com/n442\n",2);
doUrisFromInputIterator("http://bogus.com/n234\n",1);
doUrisFromInputIterator("\nhttp://bogus.com/n234",1);
doUrisFromInputIterator("\nhttp://bogus.com/n234\n",1);
}
public void doUrisFromInputIterator(String input, int expectedUris){
Iterator<String> it = new UpdateUrisInIndex.UrisFromInputIterator( new StringReader(input) );
int count = 0;
while( it.hasNext()){
String uri = it.next();
if( uri == null)
Assert.fail("UrisFromInputIterator should not return null strings \n " +
"Null string for uri #" + count + " for input '" + input + "'");
if( uri.isEmpty())
Assert.fail("UrisFromInputIterator should not return empty strings \n " +
"Empty string for uri #" + count + " for input '" + input + "'");
count++;
}
Assert.assertEquals("Incorrect number of URIs from input '" + input + "'", expectedUris, count);
}
}

9
webapp/web/WEB-INF/web.xml
View file

@ -1151,6 +1151,15 @@
<url-pattern>/admin/getObjectClasses</url-pattern>
</servlet-mapping>
<servlet>
<servlet-name>SearchServiceController</servlet-name>
<servlet-class>edu.cornell.mannlib.vitro.webapp.search.controller.SearchServiceController</servlet-class>
</servlet>
<servlet-mapping>
<servlet-name>SearchServiceController</servlet-name>
<url-pattern>/searchService/*</url-pattern>
</servlet-mapping>
<!-- VIVO OpenSocial Extension by UCSF -->
<servlet>
<servlet-name>GadgetController</servlet-name>

40
webapp/web/templates/freemarker/body/search/searchService-help.ftl Normal file
View file

@ -0,0 +1,40 @@
<#-- $This file is distributed under the terms of the license in /doc/license.txt$ -->
<h1>Search Web Service</h1>
<#if msg?has_content >
<h2>Message:</h2>
<p>${msg}</p>
</#if>
<h2>Update Search Index for URIs</h2>
<p>This service will update the search index for the list of URIs it
receives. It expectes a POST with an encoding of
multpart/form-data. The service inspect all parts of this POST for
lists of URIs to reindex. The URIs should be comma or space
seperated. If no information can be found for a URI it will be
ignored.</p>
<p>The request parameters email and password allow the form to be submitted
for a user account. Only internal accounts are supported, external authentication
is not supported.</p>
<h2>Example form for Update Search Index for URIs</h2>
<p>The following form will post to the Update URIs in search service.</p>
<form action="${urls.base}/searchService/updateUrisInSearch"
enctype="multipart/form-data"
method="post">
<label for="email">Account email</label>
<input type="text" name="email" id="email"/>
<label for="password">Account password</label>
<input type="text" name="password" id="password"/>
<label for="urisToUpdate">List of URIs to update in the search index</label>
<textarea name="urisToUpdate" id="urisToUpdate" rows="4" cols="50" ></textarea>
<button type="submit">submit</button>
</form>