[Top][All Lists]
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[Gzz-commits] storm ./Makefile org/nongnu/storm/IndexedPool.j...
|
From: |
Benja Fallenstein |
|
Subject: |
[Gzz-commits] storm ./Makefile org/nongnu/storm/IndexedPool.j... |
|
Date: |
Sat, 19 Apr 2003 08:20:33 -0400 |
CVSROOT: /cvsroot/storm
Module name: storm
Changes by: Benja Fallenstein <address@hidden> 03/04/19 08:20:32
Modified files:
. : Makefile
org/nongnu/storm: IndexedPool.java
org/nongnu/storm/impl: DirDB.java TransientPool.java
org/nongnu/storm/impl/p2p: MockP2PMap.java P2PPool.java
org/nongnu/storm/modules/gispmap: GispP2PMap.java
org/nongnu/storm/util: ContentTypeIndexType.java CopyUtil.java
Added files:
org/nongnu/storm/http: HTTPReceivedMessage.java
HTTPSendableMessage.java HTTPURL.java
MandatoryEncodings.java
ParseException.java
TransferEncodingHandler.java Util.java
org/nongnu/storm/http/client: HTTPConnection.java
HTTPRequest.java HTTPResponse.java
org/nongnu/storm/http/server: HTTPConnection.java
HTTPRequest.java HTTPResponse.java
HTTPServer.java
org/nongnu/storm/impl/p2p: P2PMap.java Peer.java
org/nongnu/storm/modules/gispmap: GispPeer.java
org/nongnu/storm/util: HTTPProxy.java UTF8Char.java
Removed files:
org/nongnu/storm/impl/p2p: MockP2PServer.java MockP2PServer.test
P2PServer.meta
Log message:
The big change: Make Storm P2P and HTTP gateway rudimentarily work.
This needs testing and polishing, and index information
is not yet put into the DHT, but-- P2P apparently **works**.
CVSWeb URLs:
http://savannah.gnu.org/cgi-bin/viewcvs/storm/storm/Makefile.diff?tr1=1.10&tr2=1.11&r1=text&r2=text
http://savannah.gnu.org/cgi-bin/viewcvs/storm/storm/org/nongnu/storm/IndexedPool.java.diff?tr1=1.3&tr2=1.4&r1=text&r2=text
http://savannah.gnu.org/cgi-bin/viewcvs/storm/storm/org/nongnu/storm/http/HTTPReceivedMessage.java?rev=1.1
http://savannah.gnu.org/cgi-bin/viewcvs/storm/storm/org/nongnu/storm/http/HTTPSendableMessage.java?rev=1.1
http://savannah.gnu.org/cgi-bin/viewcvs/storm/storm/org/nongnu/storm/http/HTTPURL.java?rev=1.1
http://savannah.gnu.org/cgi-bin/viewcvs/storm/storm/org/nongnu/storm/http/MandatoryEncodings.java?rev=1.1
http://savannah.gnu.org/cgi-bin/viewcvs/storm/storm/org/nongnu/storm/http/ParseException.java?rev=1.1
http://savannah.gnu.org/cgi-bin/viewcvs/storm/storm/org/nongnu/storm/http/TransferEncodingHandler.java?rev=1.1
http://savannah.gnu.org/cgi-bin/viewcvs/storm/storm/org/nongnu/storm/http/Util.java?rev=1.1
http://savannah.gnu.org/cgi-bin/viewcvs/storm/storm/org/nongnu/storm/http/client/HTTPConnection.java?rev=1.1
http://savannah.gnu.org/cgi-bin/viewcvs/storm/storm/org/nongnu/storm/http/client/HTTPRequest.java?rev=1.1
http://savannah.gnu.org/cgi-bin/viewcvs/storm/storm/org/nongnu/storm/http/client/HTTPResponse.java?rev=1.1
http://savannah.gnu.org/cgi-bin/viewcvs/storm/storm/org/nongnu/storm/http/server/HTTPConnection.java?rev=1.1
http://savannah.gnu.org/cgi-bin/viewcvs/storm/storm/org/nongnu/storm/http/server/HTTPRequest.java?rev=1.1
http://savannah.gnu.org/cgi-bin/viewcvs/storm/storm/org/nongnu/storm/http/server/HTTPResponse.java?rev=1.1
http://savannah.gnu.org/cgi-bin/viewcvs/storm/storm/org/nongnu/storm/http/server/HTTPServer.java?rev=1.1
http://savannah.gnu.org/cgi-bin/viewcvs/storm/storm/org/nongnu/storm/impl/DirDB.java.diff?tr1=1.3&tr2=1.4&r1=text&r2=text
http://savannah.gnu.org/cgi-bin/viewcvs/storm/storm/org/nongnu/storm/impl/TransientPool.java.diff?tr1=1.3&tr2=1.4&r1=text&r2=text
http://savannah.gnu.org/cgi-bin/viewcvs/storm/storm/org/nongnu/storm/impl/p2p/P2PMap.java?rev=1.1
http://savannah.gnu.org/cgi-bin/viewcvs/storm/storm/org/nongnu/storm/impl/p2p/Peer.java?rev=1.1
http://savannah.gnu.org/cgi-bin/viewcvs/storm/storm/org/nongnu/storm/impl/p2p/MockP2PMap.java.diff?tr1=1.2&tr2=1.3&r1=text&r2=text
http://savannah.gnu.org/cgi-bin/viewcvs/storm/storm/org/nongnu/storm/impl/p2p/P2PPool.java.diff?tr1=1.2&tr2=1.3&r1=text&r2=text
http://savannah.gnu.org/cgi-bin/viewcvs/storm/storm/org/nongnu/storm/modules/gispmap/GispPeer.java?rev=1.1
http://savannah.gnu.org/cgi-bin/viewcvs/storm/storm/org/nongnu/storm/modules/gispmap/GispP2PMap.java.diff?tr1=1.3&tr2=1.4&r1=text&r2=text
http://savannah.gnu.org/cgi-bin/viewcvs/storm/storm/org/nongnu/storm/util/HTTPProxy.java?rev=1.1
http://savannah.gnu.org/cgi-bin/viewcvs/storm/storm/org/nongnu/storm/util/UTF8Char.java?rev=1.1
http://savannah.gnu.org/cgi-bin/viewcvs/storm/storm/org/nongnu/storm/util/ContentTypeIndexType.java.diff?tr1=1.1&tr2=1.2&r1=text&r2=text
http://savannah.gnu.org/cgi-bin/viewcvs/storm/storm/org/nongnu/storm/util/CopyUtil.java.diff?tr1=1.2&tr2=1.3&r1=text&r2=text
Patches:
Index: storm/Makefile
diff -u storm/Makefile:1.10 storm/Makefile:1.11
--- storm/Makefile:1.10 Thu Apr 17 04:56:04 2003
+++ storm/Makefile Sat Apr 19 08:20:28 2003
@@ -18,6 +18,15 @@
gisp:
exec $(JAVA) $(DEBUG) org.nongnu.storm.modules.gispmap.GispP2PMap
$(SEEDS)
+http-gateway:
+ exec $(JAVA) $(DEBUG) org.nongnu.storm.util.HTTPProxy $(POOL)
+
+peer:
+ exec $(JAVA) $(DEBUG) org.nongnu.storm.modules.gispmap.GispPeer $(ARGS)
+
+fakepeer:
+ exec $(JAVA) $(DEBUG) org.nongnu.storm.impl.p2p.Peer $(POOL)
+
java:
mkdir -p CLASSES
$(JAVAC) $(DEBUG) -d $(CLASSDIR) $(RAWSRC)
Index: storm/org/nongnu/storm/IndexedPool.java
diff -u storm/org/nongnu/storm/IndexedPool.java:1.3
storm/org/nongnu/storm/IndexedPool.java:1.4
--- storm/org/nongnu/storm/IndexedPool.java:1.3 Mon Apr 7 15:50:37 2003
+++ storm/org/nongnu/storm/IndexedPool.java Sat Apr 19 08:20:28 2003
@@ -90,8 +90,8 @@
final class Mapping {
public final BlockId block;
- public final byte[] key, value;
- public Mapping(BlockId b, byte[] k, byte[] v) {
+ public final String key, value;
+ public Mapping(BlockId b, String k, String v) {
block=b; key=k; value=v;
}
}
@@ -103,7 +103,7 @@
* so code using this method must be robust
* in the face of corrupted data.
*/
- Collector get(byte[] key) throws IOException;
+ Collector get(String key) throws IOException;
}
interface IndexType {
Index: storm/org/nongnu/storm/impl/DirDB.java
diff -u storm/org/nongnu/storm/impl/DirDB.java:1.3
storm/org/nongnu/storm/impl/DirDB.java:1.4
--- storm/org/nongnu/storm/impl/DirDB.java:1.3 Tue Apr 8 04:04:45 2003
+++ storm/org/nongnu/storm/impl/DirDB.java Sat Apr 19 08:20:30 2003
@@ -36,6 +36,20 @@
*/
public class DirDB implements IndexedPool.DB {
protected File dbDir;
+
+ static final String allowed = "abcdefghijklmnopqrstuvwxyz0123456789";
+
+ static public String escape(String s) {
+ StringBuffer b = new StringBuffer();
+ for(int i=0; i<s.length(); i++) {
+ char c = s.charAt(i);
+ if(allowed.indexOf(c) >= 0)
+ b.append(c);
+ else
+ b.append("-" + ((int)c) + "-");
+ }
+ return b.toString();
+ }
/**
* @param inDir The directory of the DirPool.
@@ -47,42 +61,41 @@
public DirDB(File inDir, IndexedPool.IndexType indexType) throws
IOException {
if(inDir == null) throw new NullPointerException("null directory");
- byte[] ascii = indexType.getIndexTypeURI().getBytes("US-ASCII");
- String hex = Base32.encode(ascii);
+ String hex = escape(indexType.getIndexTypeURI());
dbDir = new File(inDir, "idx_"+hex);
if(!dbDir.exists()) {
dbDir.mkdir();
- OutputStream os = new FileOutputStream(new File(dbDir,
"index_type"));
- os.write(ascii); os.write((byte)'\n');
- os.write(indexType.getHumanReadableName().getBytes());
- os.write((byte)'\n');
- os.close();
+ Writer w = new FileWriter(new File(dbDir, "index_type"));
+ w.write(indexType.getIndexTypeURI()); w.write('\n');
+ w.write(indexType.getHumanReadableName());
+ w.write('\n');
+ w.close();
- os = new FileOutputStream(new File(dbDir, "indexed_blocks"));
- os.close();
+ w = new FileWriter(new File(dbDir, "indexed_blocks"));
+ w.close();
}
}
- protected File getKeyFile(byte[] key) throws IOException {
- return new File(dbDir, "key_"+Base32.encode(key));
+ protected File getKeyFile(String key) throws IOException {
+ return new File(dbDir, "key_"+escape(key));
}
- public Collector get(byte[] key) throws IOException {
+ public Collector get(String key) throws IOException {
Set result = new HashSet();
File f = getKeyFile(key);
if(f.exists()) {
InputStream in = new FileInputStream(getKeyFile(key));
- Reader ir = new InputStreamReader(in, "US-ASCII");
+ Reader ir = new InputStreamReader(in, "UTF-8");
BufferedReader r = new BufferedReader(ir);
String line = r.readLine();
while(line != null && !line.equals("")) {
int i = line.indexOf(' ');
BlockId block = new BlockId(line.substring(0, i));
- byte[] value = Base32.decode(line.substring(i+1));
+ String value = line.substring(i+1);
result.add(new IndexedPool.Mapping(block, key, value));
@@ -97,7 +110,7 @@
HashSet result = new HashSet();
InputStream in = new FileInputStream(new File(dbDir, "indexed_blocks"));
- Reader ir = new InputStreamReader(in, "US-ASCII");
+ Reader ir = new InputStreamReader(in, "UTF-8");
BufferedReader r = new BufferedReader(ir);
String line = r.readLine();
@@ -110,20 +123,27 @@
}
public void add(IndexedPool.Mapping m) throws IOException {
+ if(m.value.indexOf('\n') >= 0)
+ throw new UnsupportedOperationException("values containing
newlines");
+
if(getIndexed().contains(m.block))
return;
- OutputStream os = new FileOutputStream(getKeyFile(m.key).getPath(),
true);
- os.write(m.block.getURI().getBytes("US-ASCII"));
- os.write((byte)' ');
- os.write(Base32.encode(m.value).getBytes("US-ASCII"));
- os.write((byte)'\n');
- os.close();
-
- os = new FileOutputStream(new File(dbDir, "indexed_blocks").getPath(),
true);
- os.write(m.block.getURI().getBytes("US-ASCII"));
- os.write((byte)'\n');
- os.close();
+ String path = getKeyFile(m.key).getPath();
+ OutputStream os = new FileOutputStream(path, true);
+ Writer w = new OutputStreamWriter(os, "UTF-8");
+ w.write(m.block.getURI());
+ w.write(' ');
+ w.write(m.value);
+ w.write('\n');
+ w.close();
+
+ path = new File(dbDir, "indexed_blocks").getPath();
+ os = new FileOutputStream(path, true);
+ w = new OutputStreamWriter(os, "UTF-8");
+ w.write(m.block.getURI());
+ w.write('\n');
+ w.close();
}
public String toString() {
Index: storm/org/nongnu/storm/impl/TransientPool.java
diff -u storm/org/nongnu/storm/impl/TransientPool.java:1.3
storm/org/nongnu/storm/impl/TransientPool.java:1.4
--- storm/org/nongnu/storm/impl/TransientPool.java:1.3 Mon Apr 7 16:07:38 2003
+++ storm/org/nongnu/storm/impl/TransientPool.java Sat Apr 19 08:20:31 2003
@@ -81,17 +81,16 @@
protected class TransientDB implements DB {
protected Map mappings = new HashMap();
- protected Set getSet(byte[] key) {
- ByteArrayKey k = new ByteArrayKey(key);
- Set s = (Set)mappings.get(k);
+ protected Set getSet(String key) {
+ Set s = (Set)mappings.get(key);
if(s == null) {
s = new HashSet();
- mappings.put(k, s);
+ mappings.put(key, s);
}
return s;
}
- public Collector get(byte[] key) {
+ public Collector get(String key) {
return new SimpleSetCollector(new HashSet(getSet(key)));
}
Index: storm/org/nongnu/storm/impl/p2p/MockP2PMap.java
diff -u storm/org/nongnu/storm/impl/p2p/MockP2PMap.java:1.2
storm/org/nongnu/storm/impl/p2p/MockP2PMap.java:1.3
--- storm/org/nongnu/storm/impl/p2p/MockP2PMap.java:1.2 Thu Apr 17 04:36:34 2003
+++ storm/org/nongnu/storm/impl/p2p/MockP2PMap.java Sat Apr 19 08:20:32 2003
@@ -32,7 +32,7 @@
import java.io.*;
import java.util.*;
-public class MockP2PMap implements P2PPool.P2PMap {
+public class MockP2PMap implements P2PMap {
protected Map map = new HashMap();
protected Collection getSet(String key) {
Index: storm/org/nongnu/storm/impl/p2p/P2PPool.java
diff -u storm/org/nongnu/storm/impl/p2p/P2PPool.java:1.2
storm/org/nongnu/storm/impl/p2p/P2PPool.java:1.3
--- storm/org/nongnu/storm/impl/p2p/P2PPool.java:1.2 Thu Apr 17 04:36:34 2003
+++ storm/org/nongnu/storm/impl/p2p/P2PPool.java Sat Apr 19 08:20:32 2003
@@ -28,76 +28,77 @@
package org.nongnu.storm.impl.p2p;
import org.nongnu.storm.*;
import org.nongnu.storm.impl.*;
+import org.nongnu.storm.util.*;
import java.io.*;
+import java.net.*;
import java.util.*;
/** P2P-based, indexed Storm pool.
- * "Componentalized" into a "P2PMap" (an peer-to-peer indexing mechanism)
- * and a "Server" (actually a matching client and server implementation,
- * for retrieving actual files). The simplest Server impl is HTTP
- * (see <code>HttpP2PServer.py</code>, a Python module because
- * we use Python's existing HTTP server code).
+ * This pool retrieves blocks from a Storm P2P network.
+ * <p>
+ * The distributed search network is accessed through the
+ * <code>P2PMap</code> interface. Different implementations
+ * are possible.
* <p>
- * XXX finish
* XXX Become asynchronous -> don't subclass AbstractLocalPool...
*/
-public abstract class P2PPool extends AbstractLocalPool {
+public class P2PPool extends AbstractLocalPool {
+ static public boolean dbg = false;
+ static private void p(String s) { System.out.println(s); }
protected P2PMap map;
- protected Server server;
-
- /** A distributed search network.
- * Can be a DHT; can also be a Gnutella-like network
- * if desired. Abstracts over time-to-live in DHTs:
- * When an item has been <code>put()</code>,
- * this object will try to keep it published
- * in the network for its whole lifetime,
- * even if it has to re-publish it every
- * thirty minutes or so (e.g. in GISP).
- * This abstraction knows nothing about
- * actually downloading blocks.
- */
- public interface P2PMap {
- Collector get(String key) throws IOException;
- void put(String key, String value) throws IOException;
- }
-
- /** A method for sharing/downloading blocks on the network.
- * This is a server serving all blocks in a given pool
- * as well as a client for downloading blocks
- * from servers like this. This abstraction
- * knows nothing about indexing.
- * <p>
- * E.g. HTTP-based: This computer's blocks are served through HTTP,
- * and to download, HTTP connections to other servers are made.
- */
- public interface Server {
- /** Set the pool whose blocks are served
- * by this <code>Server</code>.
- * @throws IllegalStateException if this method
- * has already been called.
- */
- void setPool(StormPool pool);
-
- /** Get the URL this server offers the given block under.
- * Could be e.g. a <code>http://</code> URL. It is
- * published through the <code>P2PMap</code> so that
- * other clients with the same kind of <code>Server</code>
- * are able to download this block.
- */
- String getURL(BlockId blockId);
-
- /** Download the raw block data stored at a given URL.
- * Simply throws an IOException if the kind of URL
- * cannot be handled by this kind of server.
- */
- InputStream download(String url) throws IOException;
- }
+ protected StormPool cache;
- public P2PPool(P2PMap map, Server server,
+ public P2PPool(P2PMap map, StormPool cache,
Set indexTypes) throws IOException {
super(indexTypes);
this.map = map;
- this.server = server;
+ this.cache = cache;
+ }
+
+ public Block get(BlockId id) throws IOException {
+ try {
+ return cache.get(id);
+ } catch(FileNotFoundException _) {
+ Collection c = map.get(id.toString()).block();
+ for(Iterator i=c.iterator(); i.hasNext();) {
+ String url = (String)i.next();
+ URLConnection conn = new URL(url).openConnection();
+ BlockOutputStream bos =
+ cache.getBlockOutputStream(conn.getContentType());
+ CopyUtil.copy(conn.getInputStream(), bos);
+ if(bos.getBlockId().equals(id))
+ return bos.getBlock();
+ else
+ p("Block ids didn't match: "+id+" / "+
+ bos.getBlockId()+" (from "+url+").");
+ }
+ throw new FileNotFoundException(""+id);
+ }
+ }
+
+ public SetCollector getIds() throws IOException {
+ return cache.getIds();
+ }
+
+ public void add(Block b) {
+ throw new UnsupportedOperationException("P2PPool is retrieval only");
+ }
+
+ public void delete(Block b) {
+ throw new UnsupportedOperationException("P2PPool is retrieval only");
+ }
+
+ public BlockOutputStream getBlockOutputStream(String contentType) {
+ throw new UnsupportedOperationException("P2PPool is retrieval only");
+ }
+
+ protected DB getDB(IndexType indexType) {
+ final String prefix = indexType.getIndexTypeURI()+" ";
+ return new DB() {
+ public Collector get(String key) throws IOException {
+ return map.get(prefix + key);
+ }
+ };
}
}
Index: storm/org/nongnu/storm/modules/gispmap/GispP2PMap.java
diff -u storm/org/nongnu/storm/modules/gispmap/GispP2PMap.java:1.3
storm/org/nongnu/storm/modules/gispmap/GispP2PMap.java:1.4
--- storm/org/nongnu/storm/modules/gispmap/GispP2PMap.java:1.3 Thu Apr 17
11:16:05 2003
+++ storm/org/nongnu/storm/modules/gispmap/GispP2PMap.java Sat Apr 19
08:20:32 2003
@@ -36,14 +36,14 @@
import com.axlight.jnushare.gisp.*;
/** A <a href="http://gisp.jxta.org/";>GISP</a>-based implementation
- * of <code>P2PPool.P2PMap</code>.
+ * of <code>P2PMap</code>.
* <p>
* Note that because of license difficulties (cannot use
* JXTA-licensed code in LGPLed code because of
* the advertisement clause), this class is under the X11 license.
* That's also the main reason this is in a module.
*/
-public class GispP2PMap implements P2PPool.P2PMap {
+public class GispP2PMap implements P2PMap {
static public boolean dbg = false;
static private void pa(String s) { System.out.println(s); }
Index: storm/org/nongnu/storm/util/ContentTypeIndexType.java
diff -u storm/org/nongnu/storm/util/ContentTypeIndexType.java:1.1
storm/org/nongnu/storm/util/ContentTypeIndexType.java:1.2
--- storm/org/nongnu/storm/util/ContentTypeIndexType.java:1.1 Mon Apr 7
16:16:28 2003
+++ storm/org/nongnu/storm/util/ContentTypeIndexType.java Sat Apr 19
08:20:32 2003
@@ -50,16 +50,16 @@
* XXX For real use, this should not block.
*/
public Set getBlocks(String contentType) throws IOException {
- byte[] key = contentType.getBytes("US-ASCII");
+ String key = contentType;
Collection mappings = db.get(key).block();
Set result = new HashSet(mappings.size());
for(Iterator i=mappings.iterator(); i.hasNext();) {
IndexedPool.Mapping m = (IndexedPool.Mapping)i.next();
- if(!java.util.Arrays.equals(key, m.key))
+ if(!key.equals(m.key))
throw new Error("Key does not match");
- if(m.value.length != 0)
+ if(!m.value.equals(""))
throw new Error("Value does not match");
result.add(m.block);
@@ -70,9 +70,8 @@
}
public Set getMappings(Block block) throws IOException {
- String ct = block.getId().getContentType();
- byte[] key = ct.getBytes("US-ASCII");
- byte[] value = new byte[0];
+ String key = block.getId().getContentType();
+ String value = "";
IndexedPool.Mapping mapping = new IndexedPool.Mapping(block.getId(),
key, value);
Index: storm/org/nongnu/storm/util/CopyUtil.java
diff -u storm/org/nongnu/storm/util/CopyUtil.java:1.2
storm/org/nongnu/storm/util/CopyUtil.java:1.3
--- storm/org/nongnu/storm/util/CopyUtil.java:1.2 Thu Apr 3 09:38:14 2003
+++ storm/org/nongnu/storm/util/CopyUtil.java Sat Apr 19 08:20:32 2003
@@ -31,15 +31,31 @@
/** Utility methods for copying data from input to output streams.
*/
public class CopyUtil {
+ static private int BLOCKSIZE = 4096;
+
static public int copy(InputStream from, OutputStream to)
throws IOException {
- return copy(from, to, 4096);
+ return copy(from, to, BLOCKSIZE, true);
+ }
+
+ static public int copy(InputStream from, OutputStream to,
+ int blocksize)
+ throws IOException {
+ return copy(from, to, blocksize, true);
+ }
+
+ static public int copy(InputStream from, OutputStream to,
+ boolean close)
+ throws IOException {
+ return copy(from, to, BLOCKSIZE, close);
}
/** Copy data from an input to an output stream in blocks of a given size.
- * Closes both streams when the copy operation is complete.
+ * If <code>close</code> is true, both streams are closed
+ * when the copy operation is complete.
*/
- static public int copy(InputStream from, OutputStream to, int blockSize)
+ static public int copy(InputStream from, OutputStream to,
+ int blockSize, boolean close)
throws IOException {
try {
byte[] buf = new byte[blockSize];
@@ -57,8 +73,10 @@
//p("... all read.");
return bytesCopied;
} finally {
- from.close();
- to.close();
+ if(close) {
+ from.close();
+ to.close();
+ }
}
}
| [Prev in Thread] |
Current Thread |
[Next in Thread] |
- [Gzz-commits] storm ./Makefile org/nongnu/storm/IndexedPool.j...,
Benja Fallenstein <=