[Top][All Lists]
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[Mldonkey-commits] mldonkey distrib/ChangeLog src/daemon/common/co...
|
From: |
mldonkey-commits |
|
Subject: |
[Mldonkey-commits] mldonkey distrib/ChangeLog src/daemon/common/co... |
|
Date: |
Thu, 19 Jan 2006 02:13:19 +0000 |
CVSROOT: /sources/mldonkey
Module name: mldonkey
Branch:
Changes by: spiralvoice <address@hidden> 06/01/19 02:13:19
Modified files:
distrib : ChangeLog
src/daemon/common: commonComplexOptions.ml commonOptions.ml
commonWeb.ml
src/daemon/driver: driverMain.ml
src/utils/lib : date.ml
src/utils/net : geoip.ml http_client.ml http_client.mli
ip_set.ml
Log message:
patch #4791
CVSWeb URLs:
http://cvs.savannah.gnu.org/viewcvs/mldonkey/mldonkey/distrib/ChangeLog.diff?tr1=1.682&tr2=1.683&r1=text&r2=text
http://cvs.savannah.gnu.org/viewcvs/mldonkey/mldonkey/src/daemon/common/commonComplexOptions.ml.diff?tr1=1.47&tr2=1.48&r1=text&r2=text
http://cvs.savannah.gnu.org/viewcvs/mldonkey/mldonkey/src/daemon/common/commonOptions.ml.diff?tr1=1.120&tr2=1.121&r1=text&r2=text
http://cvs.savannah.gnu.org/viewcvs/mldonkey/mldonkey/src/daemon/common/commonWeb.ml.diff?tr1=1.23&tr2=1.24&r1=text&r2=text
http://cvs.savannah.gnu.org/viewcvs/mldonkey/mldonkey/src/daemon/driver/driverMain.ml.diff?tr1=1.89&tr2=1.90&r1=text&r2=text
http://cvs.savannah.gnu.org/viewcvs/mldonkey/mldonkey/src/utils/lib/date.ml.diff?tr1=1.9&tr2=1.10&r1=text&r2=text
http://cvs.savannah.gnu.org/viewcvs/mldonkey/mldonkey/src/utils/net/geoip.ml.diff?tr1=1.1&tr2=1.2&r1=text&r2=text
http://cvs.savannah.gnu.org/viewcvs/mldonkey/mldonkey/src/utils/net/http_client.ml.diff?tr1=1.25&tr2=1.26&r1=text&r2=text
http://cvs.savannah.gnu.org/viewcvs/mldonkey/mldonkey/src/utils/net/http_client.mli.diff?tr1=1.4&tr2=1.5&r1=text&r2=text
http://cvs.savannah.gnu.org/viewcvs/mldonkey/mldonkey/src/utils/net/ip_set.ml.diff?tr1=1.22&tr2=1.23&r1=text&r2=text
Patches:
Index: mldonkey/distrib/ChangeLog
diff -u mldonkey/distrib/ChangeLog:1.682 mldonkey/distrib/ChangeLog:1.683
--- mldonkey/distrib/ChangeLog:1.682 Thu Jan 19 02:11:14 2006
+++ mldonkey/distrib/ChangeLog Thu Jan 19 02:13:18 2006
@@ -15,6 +15,16 @@
=========
2006/01/19
+4791: web_infos: New type geoip.dat, save files to local disk,
+ re-download only if file on server is newer
+- now all files except type "rss" in web_infos are saved to local disk
+ if not present or server version is newer
+- MLDonkey uses HTML header last-modified to check if a file has been updated
+- downloaded file timestamp is changed to last-modified time
+- show number of IP blocking ranges in runinfo
+- new web_infos kind: geoip.dat
+- automatically add geoip.dat to web_infos
+- support for *.gz|bz2|zip compressed GeoIP.dat files
4811: EDK: Print warning when starting a file too big for filesystem,
clean Ux32 logging
4810: Update search results with highest avail/completesources tags (zet)
Index: mldonkey/src/daemon/common/commonComplexOptions.ml
diff -u mldonkey/src/daemon/common/commonComplexOptions.ml:1.47
mldonkey/src/daemon/common/commonComplexOptions.ml:1.48
--- mldonkey/src/daemon/common/commonComplexOptions.ml:1.47 Thu Jan 12
11:44:34 2006
+++ mldonkey/src/daemon/common/commonComplexOptions.ml Thu Jan 19 02:13:19 2006
@@ -1166,10 +1166,12 @@
^ (if Autoconf.filetp = "yes" && !!enable_fileTP then " FileTP" else
"")
^ "\nServer usage: " ^ (if !!enable_servers then "enabled" else
"disabled (you are not able to connect to ED2K Servers)")
^ (if !Geoip.active then "\nThis product includes GeoLite data created
by MaxMind, available from http://maxmind.com/"; else "")
+ ^ (let r1,r2 = Ip_set.block_stats () in if r1 = 0 then
+ "\nIP blocking disabled" else Printf.sprintf "\nIP blocking
enabled: %d ranges loaded - optimized to %d" r1 r2)
^ (let uname = Unix32.uname () in
if uname <> "" then Printf.sprintf "\nSystem info: %s" uname else "")
^ "\nUptime: " ^ Date.time_to_string (last_time () - start_time)
"verbose"
- ^ "\nLanguage: " ^ Charset.default_language
+ ^ " - Language: " ^ Charset.default_language
^ " - locale: " ^ Charset.locstr
^ " - UTC offset: " ^ Rss_date.mk_timezone (Unix.time ())
^ "\n max_string_length: " ^ string_of_int Sys.max_string_length
Index: mldonkey/src/daemon/common/commonOptions.ml
diff -u mldonkey/src/daemon/common/commonOptions.ml:1.120
mldonkey/src/daemon/common/commonOptions.ml:1.121
--- mldonkey/src/daemon/common/commonOptions.ml:1.120 Thu Jan 12 09:03:11 2006
+++ mldonkey/src/daemon/common/commonOptions.ml Thu Jan 19 02:13:19 2006
@@ -831,6 +831,8 @@
"http://www.gruk.org/server.met.gz";);
("contact.dat", 168,
"http://download.overnet.org/contact.dat";);
+ ("geoip.dat", 0,
+ "http://www.maxmind.com/download/geoip/database/GeoIP.dat.gz";);
]
let rss_feeds = define_expert_option current_section ["rss_feeds"]
@@ -855,7 +857,7 @@
);
option_hook geoip_dat (fun _ ->
try
- Geoip.init !!geoip_dat;
+ Geoip.init (Geoip.unpack !!geoip_dat);
with _ -> ()
)
@@ -1461,7 +1463,7 @@
let options_version = define_expert_option current_section ["options_version"]
"(internal option)"
- int_option 8
+ int_option 9
(*************************************************************************)
@@ -1925,4 +1927,11 @@
loop_delay =:= 20;
update 8
+ | 8 ->
+ web_infos =:= !!web_infos @ [
+ ("geoip.dat", 0,
+ "http://www.maxmind.com/download/geoip/database/GeoIP.dat.gz";);
+ ];
+ update 9
+
| _ -> ()
Index: mldonkey/src/daemon/common/commonWeb.ml
diff -u mldonkey/src/daemon/common/commonWeb.ml:1.23
mldonkey/src/daemon/common/commonWeb.ml:1.24
--- mldonkey/src/daemon/common/commonWeb.ml:1.23 Wed Dec 14 21:17:46 2005
+++ mldonkey/src/daemon/common/commonWeb.ml Thu Jan 19 02:13:19 2006
@@ -54,8 +54,56 @@
Printf.sprintf "MLDonkey/%s" Autoconf.current_version;
H.req_max_retry = 20;
} in
-
- H.wget r f
+ let r1 = {
+ r with
+ H.req_request = H.HEAD;
+ } in
+ let date = ref None in
+ begin try
+ H.whead r1 (fun headers ->
+ List.iter (fun (name, content) ->
+ if String.lowercase name = "last-modified" then
+ try
+ date := Some content
+ with _ -> ()
+ ) headers;
+ match !date with
+ None -> H.wget r f
+ | Some date ->
+ let html_time =
+ begin try
+ let t = Date.time_of_string date in
+ r.H.req_save_to_file_time <- t;
+ Unix.gmtime t
+ with e ->
+ let t = Unix.time () in
+ r.H.req_save_to_file_time <- t;
+ Unix.gmtime t
+ end
+ in
+ if not (Sys.file_exists (Filename.basename r.H.req_url.Url.short_file))
then
+ H.wget r f
+ else
+ begin
+ let file = Filename.basename r.H.req_url.Url.short_file in
+ let file_date = Unix.LargeFile.stat file in
+ let file_time = Unix.gmtime file_date.Unix.LargeFile.st_mtime in
+ if html_time <= file_time then
+ begin
+ lprintf_nl "[cWeb] using local version of %s, HTML header (%s)"
file date;
+ (f file : unit)
+ end
+ else
+ begin
+ lprintf_nl "[cWeb] downloading newer %s, HTML header (%s)"
file date;
+ H.wget r f
+ end
+ end
+ )
+ with e ->
+ lprintf_nl "[cWeb] Exception %s while loading %s"
+ (Printexc2.to_string e) url
+ end
let load_url can_fail kind url =
let f =
@@ -64,7 +112,7 @@
with e -> failwith (Printf.sprintf "Unknown kind [%s]" kind)
in
try
- lprintf_nl "[cWeb] %s loading from %s" kind url;
+ lprintf_nl "[cWeb=%s] saving %s" kind url;
mldonkey_wget url f
with e ->
if can_fail then
@@ -79,8 +127,7 @@
(List.assoc kind !file_kinds) file file
with e ->
lprintf_nl "[cWeb] Exception %s while loading kind %s"
- (Printexc2.to_string e)
- kind
+ (Printexc2.to_string e) kind
(*************************************************************************)
(* *)
@@ -316,6 +363,7 @@
let _ =
add_web_kind "rss" (fun url filename ->
let c = Rss.channel_of_file filename in
+ (try Sys.remove filename with _ -> ());
let feed =
try Hashtbl.find rss_feeds url with
Not_found ->
Index: mldonkey/src/daemon/driver/driverMain.ml
diff -u mldonkey/src/daemon/driver/driverMain.ml:1.89
mldonkey/src/daemon/driver/driverMain.ml:1.90
--- mldonkey/src/daemon/driver/driverMain.ml:1.89 Thu Jan 12 08:57:58 2006
+++ mldonkey/src/daemon/driver/driverMain.ml Thu Jan 19 02:13:19 2006
@@ -173,7 +173,9 @@
Ip_set.bl := Ip_set.load filename;
(* Ip_set.bl := Ip_set.load_merge !Ip_set.bl filename *)
CommonServer.check_blocked_servers ()
- )
+ );
+ CommonWeb.add_web_kind "geoip.dat" (fun _ filename ->
+ Geoip.init (Geoip.unpack filename))
@@ -481,7 +483,7 @@
Options.prune_file downloads_ini;
Options.prune_file users_ini;
(* Options.prune_file downloads_expert_ini; *)
- add_timer 10. (fun _ -> try CommonWeb.load_web_infos true with _ -> ());
+ add_timer 5. (fun _ -> try CommonWeb.load_web_infos true with _ -> ());
lprintf_nl () (_b "To command: telnet %s %d")
(if !!telnet_bind_addr = Ip.any then "127.0.0.1"
else Ip.to_string !!telnet_bind_addr) !!telnet_port;
Index: mldonkey/src/utils/lib/date.ml
diff -u mldonkey/src/utils/lib/date.ml:1.9 mldonkey/src/utils/lib/date.ml:1.10
--- mldonkey/src/utils/lib/date.ml:1.9 Wed Dec 21 20:43:02 2005
+++ mldonkey/src/utils/lib/date.ml Thu Jan 19 02:13:19 2006
@@ -1,3 +1,24 @@
+(* Copyright 2001, 2002 b8_bavard, b8_fee_carabine, INRIA *)
+(*
+ This file is part of mldonkey.
+
+ mldonkey is free software; you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation; either version 2 of the License, or
+ (at your option) any later version.
+
+ mldonkey is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with mldonkey; if not, write to the Free Software
+ Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+*)
+
+open Printf2
+
type date_format =
Second
| Minute
@@ -13,6 +34,7 @@
| Dot
| Minus
| Zone
+| Gmt
let months = [| "Jan"; "Feb"; "Mar"; "Apr"; "May"; "Jun";
"Jul"; "Aug"; "Sep"; "Oct"; "Nov"; "Dec"|]
@@ -39,6 +61,7 @@
| Dot -> s ^ "."
| Minus -> s ^ "-"
| Zone -> Printf.sprintf "%s%s" s (Rss_date.mk_timezone date)
+ | Gmt -> s ^ "GMT"
) "" formats
@@ -62,6 +85,42 @@
string_of_date
[WeekDay;Comma;Space;Day;Space;Month;Space;Year;Space;Hour;Colon;Minute;Colon;Second;Space;Zone]
(Unix.localtime date) date
+let apache_string date =
+ string_of_date
[WeekDay;Comma;Space;Day;Space;Month;Space;Year;Space;Hour;Colon;Minute;Colon;Second;Space;Gmt]
+ (Unix.localtime date) date
+
+let time_of_string date =
+ let month =
+ match String.sub date 8 3 with
+ "Jan" -> 0
+ | "Feb" -> 1
+ | "Mar" -> 2
+ | "Apr" -> 3
+ | "May" -> 4
+ | "Jun" -> 5
+ | "Jul" -> 6
+ | "Aug" -> 7
+ | "Sep" -> 8
+ | "Oct" -> 9
+ | "Nov" -> 10
+ | "Dec" -> 11
+ | _ -> 0
+ in
+ begin
+ try
+ let time = fst(Unix.mktime{
+ Unix.tm_isdst = false;
+ Unix.tm_wday = 0;
+ Unix.tm_yday = 0;
+ Unix.tm_sec = int_of_string (String.sub date 23 2);
+ Unix.tm_min = int_of_string (String.sub date 20 2);
+ Unix.tm_hour = int_of_string (String.sub date 17 2);
+ Unix.tm_mday = int_of_string (String.sub date 5 2);
+ Unix.tm_mon = month;
+ Unix.tm_year = int_of_string (String.sub date 12 4) - 1900}) in time
+ with e -> failwith (Printf.sprintf "date error %s" (Printexc2.to_string e))
+ end
+
let hour_in_secs = 3600
let day_in_secs = 24 * hour_in_secs
let year_in_secs = 365 * day_in_secs
Index: mldonkey/src/utils/net/geoip.ml
diff -u mldonkey/src/utils/net/geoip.ml:1.1 mldonkey/src/utils/net/geoip.ml:1.2
--- mldonkey/src/utils/net/geoip.ml:1.1 Fri Jan 6 22:23:09 2006
+++ mldonkey/src/utils/net/geoip.ml Thu Jan 19 02:13:19 2006
@@ -21,6 +21,7 @@
open Unix
open Ip
open Int64ops
+open Printf2
let country_begin = 16776960
let state_begin_rev0 = 16700000
@@ -121,6 +122,56 @@
let database_segments = ref 0
let record_length = ref 0
+let unpack filename =
+ let ext = String.lowercase (Filename2.extension filename) in
+ let last_ext = String.lowercase (Filename2.last_extension filename) in
+ let real_ext = if last_ext = ".zip" then
+ last_ext
+ else
+ ext
+ in
+ match real_ext with
+ ".zip" ->
+ begin
+ try
+ let ic = Zip.open_in filename in
+ try
+ let file = Zip.find_entry ic "GeoIP.dat" in
+ Zip.close_in ic;
+ ignore(Misc.archive_extract filename "zip");
+ (try Sys.remove "GeoIP.dat" with _ -> ());
+ Unix2.rename file.Zip.filename "GeoIP.dat";
+ "GeoIP.dat"
+ with e ->
+ Zip.close_in ic;
+ lprintf_nl "Exception %s while extracting geoip.dat"
+ (Printexc2.to_string e);
+ raise Not_found
+ with e ->
+ lprintf_nl "Exception %s while opening %s"
+ (Printexc2.to_string e) filename;
+ raise Not_found
+ end
+ | ".dat.gz" | ".dat.bz2" | ".gz" | ".bz2" ->
+ begin
+ let filetype =
+ if ext = ".bz2" || ext = ".dat.bz2" then
+ "bz2"
+ else
+ "gz"
+ in try
+ let s = Misc.archive_extract filename filetype in
+ (try Sys.remove "GeoIP.dat" with _ -> ());
+ Unix2.rename s "GeoIP.dat";
+ "GeoIP.dat"
+ with e ->
+ lprintf_nl "Exception %s while extracting"
+ (Printexc2.to_string e);
+ raise Not_found
+ end
+(* if file is not a supported archive type try loading that file anyway *)
+ | _ -> filename
+
let close () =
try
if !active then close_in !file;
@@ -198,6 +249,7 @@
end;
active := true;
+ lprintf_nl "[GeoIP] database loaded"
with _ ->
active := false
Index: mldonkey/src/utils/net/http_client.ml
diff -u mldonkey/src/utils/net/http_client.ml:1.25
mldonkey/src/utils/net/http_client.ml:1.26
--- mldonkey/src/utils/net/http_client.ml:1.25 Sun Aug 21 16:05:20 2005
+++ mldonkey/src/utils/net/http_client.ml Thu Jan 19 02:13:19 2006
@@ -44,8 +44,8 @@
req_user_agent : string;
req_accept : string;
req_proxy : (string * int) option;
-
req_url : url;
+ mutable req_save_to_file_time : float;
req_request : http_request;
req_referer : Url.url option;
req_retry : int;
@@ -62,7 +62,7 @@
let basic_request = {
req_url = Url.of_string "http://www.mldonkey.net/";;
req_referer = None;
-
+ req_save_to_file_time = 0.;
req_request = GET;
req_proxy = None;
req_headers = [];
@@ -261,13 +261,9 @@
lprintf_nl () "[%s]=[%s]" name value;
) headers;
in
- (* print_headers (); *)
+ if !verbose then print_headers ();
match ans_code with
200 ->
- (*
- lprintf "ans_code: %d\n" ans_code;
- print_headers ();
- *)
TcpBufferedSocket.set_closer sock
(fun _ _ ->
(* lprintf "default_headers_handler closer\n"; *)
@@ -391,14 +387,14 @@
lprintf_nl () "Empty content for url %s"
(Url.to_string r.req_url);
end;
- let ext = (String.lowercase (Filename2.extension
r.req_url.Url.short_file)) in
- let filename = Filename.temp_file "http_" (if ext = "" then ".tmp" else
ext) in
+ let filename = Filename.basename r.req_url.Url.short_file in
let oc = open_out_bin filename in
output_string oc s;
close_out oc;
+ if r.req_save_to_file_time <> 0. then
+ Unix.utimes filename r.req_save_to_file_time r.req_save_to_file_time;
try
- (f filename : unit);
- Sys.remove filename
+ (f filename : unit)
with e -> lprintf_nl ()
"Exception %s in loading downloaded file %s"
(Printexc2.to_string e) filename;
@@ -411,8 +407,6 @@
get_page r
(fun maxlen headers ->
- if !verbose then
- lprintf_nl () "headers...";
(try f headers with _ -> ());
fun sock nread ->
close sock Closed_by_user
Index: mldonkey/src/utils/net/http_client.mli
diff -u mldonkey/src/utils/net/http_client.mli:1.4
mldonkey/src/utils/net/http_client.mli:1.5
--- mldonkey/src/utils/net/http_client.mli:1.4 Sun Aug 21 14:26:00 2005
+++ mldonkey/src/utils/net/http_client.mli Thu Jan 19 02:13:19 2006
@@ -36,8 +36,9 @@
req_user_agent : string;
req_accept : string;
req_proxy : (string * int) option;
-
req_url : Url.url;
+ mutable req_save_to_file_time : float;
+(* re-download a saved file only if newer *)
req_request : http_request;
req_referer : Url.url option;
req_retry : int;
Index: mldonkey/src/utils/net/ip_set.ml
diff -u mldonkey/src/utils/net/ip_set.ml:1.22
mldonkey/src/utils/net/ip_set.ml:1.23
--- mldonkey/src/utils/net/ip_set.ml:1.22 Sun Dec 18 14:50:39 2005
+++ mldonkey/src/utils/net/ip_set.ml Thu Jan 19 02:13:19 2006
@@ -168,7 +168,6 @@
in bl_fold_left check () bl
let load_merge bl filename remove =
- lprintf_nl () "creating block table from %s" filename;
let guardian_regexp = Str.regexp "^\\(.*\\):
*\\([0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+\\)-\\([0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+\\)"
in
let ipfilter_regexp = Str.regexp "^\\([0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+\\)
*- *\\([0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+\\) *, *[0-9]+ *, *\\(.*\\)$" in
@@ -347,7 +346,7 @@
let nright = print_list_aux right in
nleft + 1 + nright in
- let _ = print_list_aux bl in
+ ignore(print_list_aux bl);
if !ranges_1 = 0 then
Printf.bprintf buf "%d ranges loaded\n" !ranges_1
else
@@ -360,6 +359,8 @@
None -> false
| Some br -> true
+let block_stats () = (!ranges_1, !ranges_2)
+
(*
open Benchmark
- [Mldonkey-commits] mldonkey distrib/ChangeLog src/daemon/common/co..., (continued)
- [Mldonkey-commits] mldonkey distrib/ChangeLog src/daemon/common/co..., mldonkey-commits, 2006/01/06
- [Mldonkey-commits] mldonkey distrib/ChangeLog src/daemon/common/co..., mldonkey-commits, 2006/01/07
- [Mldonkey-commits] mldonkey distrib/ChangeLog src/daemon/common/co..., mldonkey-commits, 2006/01/08
- [Mldonkey-commits] mldonkey distrib/ChangeLog src/daemon/common/co..., mldonkey-commits, 2006/01/08
- [Mldonkey-commits] mldonkey distrib/ChangeLog src/daemon/common/co..., mldonkey-commits, 2006/01/11
- [Mldonkey-commits] mldonkey distrib/ChangeLog src/daemon/common/co..., mldonkey-commits, 2006/01/11
- [Mldonkey-commits] mldonkey distrib/ChangeLog src/daemon/common/co..., mldonkey-commits, 2006/01/12
- [Mldonkey-commits] mldonkey distrib/ChangeLog src/daemon/common/co..., mldonkey-commits, 2006/01/12
- [Mldonkey-commits] mldonkey distrib/ChangeLog src/daemon/common/co..., mldonkey-commits, 2006/01/16
- [Mldonkey-commits] mldonkey distrib/ChangeLog src/daemon/common/co..., mldonkey-commits, 2006/01/18
- [Mldonkey-commits] mldonkey distrib/ChangeLog src/daemon/common/co...,
mldonkey-commits <=
- [Mldonkey-commits] mldonkey distrib/ChangeLog src/daemon/common/co..., mldonkey-commits, 2006/01/29
- [Mldonkey-commits] mldonkey distrib/ChangeLog src/daemon/common/co..., mldonkey-commits, 2006/01/29
- [Mldonkey-commits] mldonkey distrib/ChangeLog src/daemon/common/co..., mldonkey-commits, 2006/01/29