[Top][All Lists]
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[Gzz-commits] storm/org/nongnu/storm modules/gispmap/GispPeer...
|
From: |
Benja Fallenstein |
|
Subject: |
[Gzz-commits] storm/org/nongnu/storm modules/gispmap/GispPeer... |
|
Date: |
Sat, 03 May 2003 15:45:26 -0400 |
CVSROOT: /cvsroot/storm
Module name: storm
Changes by: Benja Fallenstein <address@hidden> 03/05/03 15:45:26
Modified files:
org/nongnu/storm/modules/gispmap: GispPeer.java
org/nongnu/storm/util: HTTPProxy.java
Added files:
org/nongnu/storm/util: HtmlLinkIndex.java
Log message:
A Storm index that allows finding backlinks
to HTML pages in Storm. That should explain
what indexing is for & why it's important
to hypertext, much easier than explaining
Xanalogical storage... :-)
CVSWeb URLs:
http://savannah.gnu.org/cgi-bin/viewcvs/storm/storm/org/nongnu/storm/modules/gispmap/GispPeer.java.diff?tr1=1.3&tr2=1.4&r1=text&r2=text
http://savannah.gnu.org/cgi-bin/viewcvs/storm/storm/org/nongnu/storm/util/HtmlLinkIndex.java?rev=1.1
http://savannah.gnu.org/cgi-bin/viewcvs/storm/storm/org/nongnu/storm/util/HTTPProxy.java.diff?tr1=1.15&tr2=1.16&r1=text&r2=text
Patches:
Index: storm/org/nongnu/storm/modules/gispmap/GispPeer.java
diff -u storm/org/nongnu/storm/modules/gispmap/GispPeer.java:1.3
storm/org/nongnu/storm/modules/gispmap/GispPeer.java:1.4
--- storm/org/nongnu/storm/modules/gispmap/GispPeer.java:1.3 Fri May 2
16:28:24 2003
+++ storm/org/nongnu/storm/modules/gispmap/GispPeer.java Sat May 3
15:45:26 2003
@@ -30,6 +30,7 @@
import org.nongnu.storm.impl.*;
import org.nongnu.storm.impl.p2p.*;
import org.nongnu.storm.util.HTTPProxy;
+import org.nongnu.storm.util.HtmlLinkIndex;
import java.io.*;
import java.util.*;
@@ -52,6 +53,7 @@
IndexedPool publishedPool = null;
int gisp_port = GispP2PMap.PORT;
int gateway_port = -1;
+ Set indexTypes = Collections.singleton(HtmlLinkIndex.type);
int i=0;
while(i<argv.length) {
@@ -59,7 +61,7 @@
i++;
p("Publish pool: "+argv[i]);
publishedPool = new DirPool(new File(argv[i]),
- Collections.EMPTY_SET);
+ indexTypes);
} else if(argv[i].equals("-gw")) {
i++;
gateway_port = Integer.parseInt(argv[i]);
@@ -80,7 +82,7 @@
System.arraycopy(argv, i, seeds, 0, seeds.length);
GispPeer peer = new GispPeer(publishedPool,
- new TransientPool(Collections.EMPTY_SET),
+ new TransientPool(indexTypes),
gisp_port, seeds);
if(gisp_port == GispP2PMap.PORT)
p("GISP UDP port: default of "+gisp_port);
Index: storm/org/nongnu/storm/util/HTTPProxy.java
diff -u storm/org/nongnu/storm/util/HTTPProxy.java:1.15
storm/org/nongnu/storm/util/HTTPProxy.java:1.16
--- storm/org/nongnu/storm/util/HTTPProxy.java:1.15 Fri May 2 09:44:48 2003
+++ storm/org/nongnu/storm/util/HTTPProxy.java Sat May 3 15:45:26 2003
@@ -34,15 +34,16 @@
import java.io.*;
import java.net.*;
import java.util.Iterator;
+import java.util.NoSuchElementException;
/** An HTTP server serving blocks from a Storm pool. When started from command
* line, it servers the directory given as first argument.
*/
public class HTTPProxy implements Runnable {
public static boolean dbg = false;
- private static void p(String s) { System.out.println(s); }
+ private static void p(String s) { System.out.println("HTTPProxy: "+s); }
- protected StormPool pool;
+ protected IndexedPool pool;
protected int port;
protected String addr;
protected String url;
@@ -51,7 +52,7 @@
protected HTTPServer server;
- public HTTPProxy(StormPool pool, int port) throws IOException {
+ public HTTPProxy(IndexedPool pool, int port) throws IOException {
this.pool = pool;
this.port = port;
this.addr = InetAddress.getLocalHost().getCanonicalHostName();
@@ -84,7 +85,7 @@
if(HTTPProxy.dbg) p("<"+port+"> GET: "+uri);
if(uri.startsWith("/")) uri = uri.substring(1);
- boolean rewrite = false;
+ boolean rewrite = false, backlinks = false;
if(uri.startsWith("rewrite/")) {
uri = uri.substring("rewrite/".length());
rewrite = true;
@@ -92,6 +93,11 @@
return makePAC(req, resf);
}
+ if(uri.startsWith("backlinks/")) {
+ uri = uri.substring("backlinks/".length());
+ backlinks = true;
+ }
+
if(uri.equals("") || uri.equals("rewrite")) {
if(uri.equals("rewrite")) rewrite = true;
@@ -121,18 +127,70 @@
if(HTTPProxy.dbg) p("<"+port+"> Serve: "+id);
HTTPResponse resp = resf.makeResponse(200, "Ok");
resp.setField("Content-Type", id.getContentType());
- if(!rewrite || !id.getContentType().equals("text/html")) {
+ if((!rewrite && !backlinks) ||
+ !id.getContentType().equals("text/html")) {
CopyUtil.copy(block.getInputStream(),
resp.getOutputStream());
} else {
+ if(HTTPProxy.dbg) p("Start rewrite");
+
String s = CopyUtil.readString(block.getInputStream());
- String l = s.toLowerCase();
- int i = -1;
- while((i=l.indexOf("urn:x-storm:1.0:", i)) >= 0) {
- s = s.substring(0, i) + REWRITE + s.substring(i);
- l = s.toLowerCase();
- i += REWRITE.length() + 1;
+
+ String prefix = "";
+
+ if(rewrite) {
+ prefix = REWRITE;
+ if(backlinks) prefix += "backlinks/";
+
+ String l = s.toLowerCase();
+ int i = -1;
+ while((i=l.indexOf("urn:x-storm:1.0:", i)) >= 0) {
+ s = s.substring(0, i) + prefix + s.substring(i);
+ l = s.toLowerCase();
+ i += prefix.length() + 1;
+ }
+ }
+
+ if(backlinks) {
+ if(HTTPProxy.dbg) p("Getting HtmlLinkIndex.");
+
+ HtmlLinkIndex idx = null;
+ try {
+ idx =
(HtmlLinkIndex)pool.getIndex(HtmlLinkIndex.uri);
+ } catch(NoSuchElementException _) {}
+
+ if(HTTPProxy.dbg) p("HtmlLinkIndex = "+idx);
+
+ if(idx != null) {
+ if(HTTPProxy.dbg) p("Looking for links");
+ Collector links = idx.getLinksTo(id);
+
+ try {
+ Thread.sleep(2000);
+ } catch(InterruptedException _) {}
+
+ if(HTTPProxy.dbg) p("Iter thru links");
+ String t = null;
+ synchronized(links) {
+ for(Iterator iter=links.iterator();
+ iter.hasNext();) {
+ HtmlLinkIndex.Link link =
+ (HtmlLinkIndex.Link)iter.next();
+ if(HTTPProxy.dbg) p("Link: "+link.linkText);
+ if(t == null) t = "";
+ t += "[<a
href=\""+prefix+link.linkFrom.getURI()+
+ "\">"+link.linkText+"</a>]<br>";
+ }
+ }
+ if(t != null) {
+ int i = s.indexOf("</body>");
+ if(i < 0) i = s.length();
+ s = s.substring(0, i) +
"<p><tr><td><small>Backlinks:<br>" +
+ t + "</small></p>" + s.substring(i);
+ }
+ }
}
+
byte[] bytes = s.getBytes("US-ASCII");
resp.getOutputStream().write(bytes);
}
@@ -202,7 +260,7 @@
public static void main(String[] args) throws Exception {
dbg = true;
- StormPool pool;
+ IndexedPool pool;
java.util.Set empty = java.util.Collections.EMPTY_SET;
pool = new DirPool(new File(args[0]), empty);