* Listens to tag-value events and passes on indexing events to an IndexStore. * This is an update to Indexer that understands that indexed properties may * not be at the top level. *
* ** This class is provided to allow the indexing of arbitrary record-based text * files. Indexer objects are built for a single file and the indexes are * written to a single index store. To keep all of the reader offsets in sync * with one another, you will almost certainly wish to use the getReader() * method to retrieve a CountedBufferedReader instance if you want to read the * byte-offset between calls to Parser.read(). Below is an example of how to * index a file. *
* *
* File fileToIndex; // get this from somewhere
*
* // don't forget to register all the apropreate keys to the factory first.
* BioIndexStore indexStore = bioIndxStrFact.createBioStore();
*
* Indexer indexer = new Indexer(fileToIndex, indexStore);
* indexer.setPrimaryKeyName("foo", new String[] { "foo" });
* indexer.addSecondaryKey("bar", new String[] { "x", "y", "bar"});
* indexer.addSecondaryKey("baz", new String[] { "z" });
*
* TagValueParser tvParser; // make this appropriate for your format
* TagValueListener listener; // make this appropriate for your format
* // and forward all events to indexer
*
* Parser parser = new Parser();
* while(
* parser.read(indexer.getReader(), tvParser, listener)
* ) {
* System.out.print(".");
* }
*
*
* @since 1.2
* @author Matthew Pocock
*/
public class Indexer2
implements TagValueListener {
private final String primaryKeyName;
private final RAF file;
private final CountedBufferedReader reader;
private final IndexStore indexStore;
private final Map keys;
private final Map keyValues;
private Object tag;
private long offset;
private int depth;
private Stack stack;
/**
* Build a new Indexer.
*
* @param file the file to be processed
* @param indexStore the IndexStore to write to
*/
public Indexer2(File file, IndexStore indexStore, Index2Model model)
throws FileNotFoundException {
this.file = new RAF(file, "r");
this.reader = new CountedBufferedReader(new FileReader(file));
this.indexStore = indexStore;
this.keyValues = new SmallMap();
this.depth = 0;
this.stack = new Stack();
this.keys = new SmallMap();
for(Iterator i = model.getKeys().iterator(); i.hasNext(); ) {
String key = (String) i.next();
Object val = model.getKeyPath(key);
keys.put(val, key);
}
this.primaryKeyName = model.getPrimaryKeyName();
}
/**
* Retrieve the reader that can be safely used to index this file.
*
* @return the CountedBufferedReader that should be processed
*/
public CountedBufferedReader getReader() {
return reader;
}
public void startRecord() {
if(depth == 0) {
offset = reader.getFilePointer();
Frame frame = new Frame();
for(Iterator ki = keys.keySet().iterator(); ki.hasNext(); ) {
Object[] keyPath = (Object[]) ki.next();
if(keyPath.length == 1) {
frame.addKey(keyPath);
} else {
frame.paths.add(keyPath);
}
}
stack.push(frame);
} else {
Frame top = (Frame) stack.peek();
Frame frame = new Frame();
//System.out.println("Tag: " + tag);
//System.out.println("Deth: " + depth);
//System.out.println("Top: " + top);
for(Iterator ki = top.paths.iterator(); ki.hasNext(); ) {
Object[] keyPath = (Object[]) ki.next();
if(keyPath[depth-1].equals(tag)) {
if((keyPath.length-1) == depth) {
frame.addKey(keyPath);
} else {
frame.paths.add(keyPath);
}
}
}
//System.out.println("Pushing new stack frame: " + top + " <- " + frame);
stack.push(frame);
}
depth++;
}
public void startTag(Object tag) {
this.tag = tag;
//if(depth >= 2) System.out.println("tag: " + tag);
}
public void value(TagValueContext ctxt, Object value) {
Frame frame = (Frame) stack.peek();
Object[] keyPath = (Object []) frame.getKeyPath(tag);
if(keyPath != null) {
//if(depth >= 2) System.out.println("Interested in: " + tag + " -> " + value);
KeyState ks = (KeyState) keyValues.get(keyPath);
if(ks == null) {
//if(depth >= 2) System.out.println("Allocating stoorage");
keyValues.put(keyPath, ks = new KeyState(keys.get(keyPath).toString()));
}
ks.values.add(value);
//if(depth >= 2) System.out.println(keyValues);
}
}
public void endTag() {}
public void endRecord()
throws ParserException
{
depth--;
if(depth == 0) {
int length = (int) (reader.getFilePointer() - offset);
//System.out.println("keyValues: " + keyValues);
String primaryKeyValue = null;
Map secKeys = new SmallMap();
for(Iterator i = keyValues.keySet().iterator(); i.hasNext(); ) {
Object key = i.next();
KeyState ks = (KeyState) keyValues.get(key);
if(ks.keyName.equals(primaryKeyName)) {
if(ks.values.size() != 1) {
throw new ParserException(
"There must be exactly one value for the primary key: " +
primaryKeyName + " - " + ks.values
);
}
primaryKeyValue = ks.values.iterator().next().toString();
} else {
secKeys.put(ks.keyName, ks.values);
}
}
if(primaryKeyValue == null) {
throw new NullPointerException("No primary key");
}
//System.out.println("Primary: " + primaryKeyValue);
//System.out.println("Secondaries: " + secKeys);
indexStore.writeRecord(
file,
offset,
length,
primaryKeyValue,
secKeys
);
stack.clear();
for(Iterator i = keyValues.values().iterator(); i.hasNext(); ) {
KeyState ks = (KeyState) i.next();
ks.values.clear();
}
} else {
stack.pop();
}
}
private static class Frame {
public final Map keys = new SmallMap();
public final Set paths = new SmallSet();
public void addKey(Object[] keyPath) {
keys.put(keyPath[keyPath.length - 1], keyPath);
}
public Object[] getKeyPath(Object tag) {
return (Object []) keys.get(tag);
}
public String toString() {
return this.getClass() + ": (" + keys + "\t" + paths + " )";
}
}
private static class KeyState {
public final String keyName;
public final Set values = new SmallSet();
public KeyState(String keyName) {
this.keyName = keyName;
}
public String toString() {
return this.getClass() + ": (" + keyName + " " + values + ")";
}
}
}