[Top][All Lists]
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]
[Mldonkey-commits] mldonkey distrib/ChangeLog src/networks/donkey/...
|
From: |
mldonkey-commits |
|
Subject: |
[Mldonkey-commits] mldonkey distrib/ChangeLog src/networks/donkey/... |
|
Date: |
Wed, 31 May 2006 20:02:14 +0000 |
CVSROOT: /sources/mldonkey
Module name: mldonkey
Branch:
Changes by: spiralvoice <address@hidden> 06/05/31 20:02:14
Modified files:
distrib : ChangeLog
src/networks/donkey: donkeyClient.ml donkeyGlobals.ml
donkeyShare.ml donkeyTypes.ml
tools : mld_hash.ml
Log message:
patch #5145
CVSWeb URLs:
http://cvs.savannah.gnu.org/viewcvs/mldonkey/mldonkey/distrib/ChangeLog.diff?tr1=1.861&tr2=1.862&r1=text&r2=text
http://cvs.savannah.gnu.org/viewcvs/mldonkey/mldonkey/src/networks/donkey/donkeyClient.ml.diff?tr1=1.91&tr2=1.92&r1=text&r2=text
http://cvs.savannah.gnu.org/viewcvs/mldonkey/mldonkey/src/networks/donkey/donkeyGlobals.ml.diff?tr1=1.81&tr2=1.82&r1=text&r2=text
http://cvs.savannah.gnu.org/viewcvs/mldonkey/mldonkey/src/networks/donkey/donkeyShare.ml.diff?tr1=1.44&tr2=1.45&r1=text&r2=text
http://cvs.savannah.gnu.org/viewcvs/mldonkey/mldonkey/src/networks/donkey/donkeyTypes.ml.diff?tr1=1.40&tr2=1.41&r1=text&r2=text
http://cvs.savannah.gnu.org/viewcvs/mldonkey/mldonkey/tools/mld_hash.ml.diff?tr1=1.7&tr2=1.8&r1=text&r2=text
Patches:
Index: mldonkey/distrib/ChangeLog
diff -u mldonkey/distrib/ChangeLog:1.861 mldonkey/distrib/ChangeLog:1.862
--- mldonkey/distrib/ChangeLog:1.861 Tue May 30 11:48:02 2006
+++ mldonkey/distrib/ChangeLog Wed May 31 20:02:14 2006
@@ -14,6 +14,10 @@
ChangeLog
=========
+2006/05/31
+5145: EDK: Fix md4 hashing when filesize is an exact multiple
+ of edk chunk size, MLDonkey is now eMule-compatible (zet)
+
2006/05/30
5141: Swarmer: New option swarming_block_selection_algorithm (pango)
* Allow to choose swarmer block selection algorithm at runtime.
Index: mldonkey/src/networks/donkey/donkeyClient.ml
diff -u mldonkey/src/networks/donkey/donkeyClient.ml:1.91
mldonkey/src/networks/donkey/donkeyClient.ml:1.92
--- mldonkey/src/networks/donkey/donkeyClient.ml:1.91 Thu May 25 19:47:25 2006
+++ mldonkey/src/networks/donkey/donkeyClient.ml Wed May 31 20:02:14 2006
@@ -877,21 +877,22 @@
end;
let chunks =
- if file_size file <= block_size then Bitv.create 1 true
- else
- if Bitv.length chunks = 0 then
- Bitv.create file.file_nchunks true
- else
- if Bitv.length chunks <> file.file_nchunks then begin
- if !verbose then
- lprintf_nl "number of chunks is different %d/%d for %s(%s), size %Ld on
%s"
- (Bitv.length chunks)
- file.file_nchunks
- (file_best_name file)
- (Md4.to_string file.file_md4)
- (file_size file)
- (full_client_identifier c);
- Bitv.create file.file_nchunks false
+ if file_size file <= block_size
+ then Bitv.create 1 true
+ else
+ if Bitv.length chunks = 0
+ then Bitv.create file.file_nchunks true
+ else
+ if Bitv.length chunks <> file.file_nchunks then begin
+ if !verbose then
+ lprintf_nl "number of chunks is different %d/%d for %s(%s),
size %Ld on %s"
+ (Bitv.length chunks)
+ (file.file_nchunks)
+ (file_best_name file)
+ (Md4.to_string file.file_md4)
+ (file_size file)
+ (full_client_identifier c);
+ Bitv.create file.file_nchunks false
(* What should we do ?
1) Try to recover the correct size of the file: we can use
@@ -901,10 +902,10 @@
*)
- end else
- chunks in
- if is_useful_client file chunks then client_is_useful c file chunks
+ end else chunks
+ in
+ if is_useful_client file chunks then client_is_useful c file chunks
let send_pending_messages c sock =
let module M = DonkeyProtoClient in
@@ -1464,9 +1465,9 @@
(file_best_name file) (full_client_identifier c);
if file.file_computed_md4s = [||] then begin
- if file.file_nchunks = 1 then begin
+ if file.file_nchunk_hashes = 0 then begin
lprintf_nl "[ERROR] file %s has only one chunk, ignoring
QueryChunkMd4ReplyReq"
- (file_best_name file);
+ (file_best_name file);
file.file_computed_md4s <- [|file.file_md4|];
match file.file_swarmer with
None -> ()
@@ -1474,19 +1475,19 @@
CommonSwarming.set_verifier swarmer
(Verification [| Ed2k file.file_md4 |])
end else
- if t.Q.chunks = [||] then
- lprintf_nl "[ERROR] received empty chunks md4 message for %s from
%s"
- (file_best_name file) (full_client_identifier c)
- else
- if Array.length t.Q.chunks <> file.file_nchunks then begin
- if !verbose then
- lprintf_nl "[ERROR] number of chunks does not match, received md4s
%d/should be %d, for %s(%s):%Ld bytes from %s"
- (Array.length t.Q.chunks)
- file.file_nchunks
- (file_best_name file)
- (Md4.to_string file.file_md4)
- (file_size file)
- (full_client_identifier c)
+ if t.Q.chunks = [||] then
+ lprintf_nl "[ERROR] received empty chunks md4 message for %s
from %s"
+ (file_best_name file) (full_client_identifier c)
+ else
+ if Array.length t.Q.chunks <> file.file_nchunk_hashes then begin
+ if !verbose then
+ lprintf_nl "[ERROR] number of chunks does not match, received
md4s %d/should be %d, for %s(%s):%Ld bytes from %s"
+ (Array.length t.Q.chunks)
+ (file.file_nchunks)
+ (file_best_name file)
+ (Md4.to_string file.file_md4)
+ (file_size file)
+ (full_client_identifier c)
(* What should we do ?
1) Try to recover the correct size of the file: we can use
@@ -1499,14 +1500,15 @@
*)
- end else begin
+ end else begin
(* We should check the correctness of the Md4 array *)
-
- let md4s = t.Q.chunks in
- let md4 = DonkeyShare.md4_of_array md4s in
- if md4 <> file.file_md4 then begin
- lprintf_nl "[ERROR] Chunks md4s do not match file_md4 for
%s(%s) from %s"
- (file_best_name file) (Md4.to_string file.file_md4)
(full_client_identifier c);
+ let md4s = t.Q.chunks in
+ let md4 = DonkeyShare.md4_of_array md4s in
+ if md4 <> file.file_md4 then begin
+ lprintf_nl "[ERROR] Chunks md4s do not match file_md4 for
%s(%s) from %s"
+ (file_best_name file)
+ (Md4.to_string file.file_md4)
+ (full_client_identifier c);
end else begin
file.file_computed_md4s <- md4s;
match file.file_swarmer with
@@ -1936,8 +1938,8 @@
match file.file_swarmer with
None ->
(* file was found, if we have no swarmer, we have
- the file complete and share it! it's save to
- asume that we have all chunks! *)
+ the file complete and share it! it's safe to
+ assume that we have all chunks! *)
Bitv.create file.file_nchunks true
| Some swarmer ->
let bitmap = CommonSwarming.chunks_verified_bitmap swarmer in
@@ -2610,7 +2612,7 @@
(CommonClient.as_client c.client_client);
with
- Not_found -> ()
+ Not_found -> ()
| e ->
if !verbose then
lprintf_nl "add_location: exception %s" (Printexc2.to_string e)
@@ -2625,7 +2627,7 @@
(CommonClient.as_client c.client_client);
with
- Not_found -> ()
+ Not_found -> ()
| e ->
if !verbose then
lprintf_nl "remove_location for file_md4 %s: exception %s"
Index: mldonkey/src/networks/donkey/donkeyGlobals.ml
diff -u mldonkey/src/networks/donkey/donkeyGlobals.ml:1.81
mldonkey/src/networks/donkey/donkeyGlobals.ml:1.82
--- mldonkey/src/networks/donkey/donkeyGlobals.ml:1.81 Thu May 25 19:47:25 2006
+++ mldonkey/src/networks/donkey/donkeyGlobals.ml Wed May 31 20:02:14 2006
@@ -145,6 +145,29 @@
let zone_size = Int64.of_int (180 * 1024)
let block_size = 9728000L
+(* Old value: *)
+(* let nchunks = Int64.to_int (Int64.pred file_size // block_size) + 1 in *)
+
+(* New value: *)
+(* From Emule: KnownFile.cpp
+// File size Data parts ED2K parts ED2K part hashs
+// ---------------------------------------------------------------
+// 1..PARTSIZE-1 1 1 0(!)
+// PARTSIZE 1 2(!) 2(!)
+// PARTSIZE+1 2 2 2
+// PARTSIZE*2 2 3(!) 3(!)
+// PARTSIZE*2+1 3 3 3
+*)
+
+let get_nchunks size = Int64.to_int (size // block_size) + 1
+
+let get_nchunk_hashes size =
+ let nchunk_hashes = Int64.to_int (size // block_size) in
+ let nchunk_hashes = if nchunk_hashes <> 0
+ then nchunk_hashes + 1
+ else nchunk_hashes in
+ nchunk_hashes
+
let queue_timeout = ref (60. *. 10.) (* 10 minutes *)
let files_queries_per_minute = 3 (* queries for 3 files cost 3*16=48
server-credits; we did get 60 (1 each second) *)
@@ -343,30 +366,28 @@
if file_size <> zero && writable then (* do not truncate if not writable
*)
begin
- try
+ try
Unix32.ftruncate64 t file_size !!create_file_sparse
- with e ->
- (try
- Unix32.remove t
- with e ->
+ with e ->
+ (try
+ Unix32.remove t
+ with e ->
lprintf_nl "Unix32.remove %s exception %s"
- (file_diskname) (Printexc2.to_string e));
- Unix32.destroy t;
- failwith (Printf.sprintf "file size %s is too big, exception: %s"
- (size_of_int64 file_size) (Printexc2.to_string e))
- end;
-
- let nchunks = Int64.to_int (Int64.pred file_size // block_size) + 1 in
- let md4s = if file_size <= block_size then
- [md4]
- else [] in
+ (file_diskname) (Printexc2.to_string e));
+ Unix32.destroy t;
+ failwith (Printf.sprintf "file size %s is too big, exception: %s"
+ (size_of_int64 file_size) (Printexc2.to_string e))
+ end;
+
+ let md4s = if file_size < block_size then [md4] else [] in
let rec file = {
file_diskname = file_diskname;
file_file = file_impl;
file_shared = None;
file_md4 = md4;
file_swarmer = None;
- file_nchunks = nchunks;
+ file_nchunks = get_nchunks file_size;
+ file_nchunk_hashes = get_nchunk_hashes file_size;
file_filenames = filenames;
file_computed_md4s = Array.of_list md4s;
file_format = FormatNotComputed 0;
@@ -960,22 +981,22 @@
key_checked := true;
if not (try String.sub !!client_private_key 0 4 = "MIIB" with e ->
false) then
if !key_check_again then
- begin
+ begin
lprintf_nl "can not create proper client_private_key, exiting...";
- exit 70
- end
- else
+ exit 70
+ end
+ else
begin
lprintf_nl "bad client_private_key detected, creating new key";
- set_simple_option donkey_ini "client_private_key"
(DonkeySui.SUI.create_key ());
- key_check_again := true
- end
+ set_simple_option donkey_ini "client_private_key"
(DonkeySui.SUI.create_key ());
+ key_check_again := true
+ end
in
if not !key_checked then check_client_private_key ();
if !key_check_again then
begin
lprintf_nl "re-checking private key";
- check_client_private_key ()
+ check_client_private_key ()
end
);
end
Index: mldonkey/src/networks/donkey/donkeyShare.ml
diff -u mldonkey/src/networks/donkey/donkeyShare.ml:1.44
mldonkey/src/networks/donkey/donkeyShare.ml:1.45
--- mldonkey/src/networks/donkey/donkeyShare.ml:1.44 Thu May 25 19:47:25 2006
+++ mldonkey/src/networks/donkey/donkeyShare.ml Wed May 31 20:02:14 2006
@@ -125,16 +125,16 @@
*)
match file.file_swarmer with
Some s -> (let len = Array.length md4s in
- let ver_str = String.make len '3' in
- CommonSwarming.set_chunks_verified_bitmap s ver_str;
- (*
- CommonSwarming.set_present s [(Int64.zero, file_size file)];
- (* If we don't verify now, it will never happen! *)
- CommonSwarming.verify_all_blocks s true;
- *)
+ let ver_str = String.make len '3' in
+ CommonSwarming.set_chunks_verified_bitmap s ver_str;
+ (*
+ CommonSwarming.set_present s [(Int64.zero, file_size file)];
+ (* If we don't verify now, it will never happen! *)
+ CommonSwarming.verify_all_blocks s true;
+ *)
if !verbose_share then
lprintf_nl "verified map of %s = %s"
- (codedname) (CommonSwarming.chunks_verified_bitmap s))
+ (codedname) (CommonSwarming.chunks_verified_bitmap s))
| None -> if !verbose_share then lprintf_nl "no swarmer for %s"
codedname;
(try
file.file_format <- CommonMultimedia.get_info
@@ -181,14 +181,15 @@
begin
if !verbose_share || !verbose then
lprintf_nl "send_new_shared: found master server %s:%d"
- (Ip.to_string s.server_ip) s.server_port;
- tag := true;
+ (Ip.to_string s.server_ip) s.server_port;
+ tag := true;
do_if_connected s.server_sock (fun sock ->
server_send_share s.server_has_zlib sock !new_shared_files)
- end) (connected_servers ());
+ end
+ ) (connected_servers ());
if !tag && (!verbose_share || !verbose) then
lprintf_nl "send_new_shared: Sent %d new shared files to servers"
- (List.length !new_shared_files);
+ (List.length !new_shared_files);
new_shared_files := []
end
else
@@ -203,12 +204,12 @@
exception Wrong_file_size
-let computation = ref false
+let computing = ref false
(* Compute (at most) one MD4 chunk if needed. *)
let check_shared_files () =
let module M = CommonHasher in
- if not !computation then
+ if not !computing then
match !shared_files with
[] -> ()
| sh :: files ->
@@ -218,48 +219,50 @@
raise Not_found;
if Unix32.getsize sh.shared_name <> sh.shared_size then
raise Wrong_file_size;
- computation := true;
+
+ computing := true;
+
let end_pos = sh.shared_pos ++ block_size in
- let end_pos = if end_pos > sh.shared_size then sh.shared_size
- else end_pos in
+ let end_pos = min end_pos sh.shared_size in
let len = end_pos -- sh.shared_pos in
+
if !verbose_md4 then
- lprintf_nl "Hash chunk %Ld/%Ld of %s"
- ((Int64.div sh.shared_pos block_size) ++ one)
- ((Int64.div sh.shared_size block_size) ++ one)
- sh.shared_name;
+ lprintf_nl "Hashing chunk %d: %Ld-%Ld (%Ld) of %s"
+ sh.shared_chunk sh.shared_pos end_pos len sh.shared_name;
M.compute_md4 (Unix32.filename sh.shared_fd) sh.shared_pos len
(fun job ->
- computation := false;
+ computing := false;
if job.M.job_error then begin
- lprintf_nl "Error prevent sharing %s" sh.shared_name
- end else
- let _ = () in
- let new_md4 = job.M.job_result in
-
- sh.shared_list <- new_md4 :: sh.shared_list;
- sh.shared_pos <- end_pos;
- if end_pos = sh.shared_size then begin
+ lprintf_nl "Error prevent sharing %s" sh.shared_name
+ end else
+ let new_md4 = job.M.job_result in
+
+ sh.shared_list <- new_md4 :: sh.shared_list;
+ sh.shared_pos <- end_pos;
+ sh.shared_chunk <- sh.shared_chunk + 1;
+
+ if sh.shared_chunk = get_nchunks sh.shared_size then begin
let s = {
sh_name = sh.shared_name;
sh_size = sh.shared_size;
sh_md4s = Array.of_list (List.rev sh.shared_list);
sh_mtime = Unix32.mtime sh.shared_name;
- } in
+ } in
lprintf_nl "New shared file %s" sh.shared_name;
Hashtbl.add shared_files_info
(s.sh_name, s.sh_size, s.sh_mtime) s;
known_shared_files =:= s :: !!known_shared_files;
- shared_files := files;
+ shared_files := files;
new_file_to_share s sh.shared_shared.impl_shared_codedname
(Some sh.shared_shared);
end
else
- job_creater ())
+ job_creater ()
+ )
with e ->
- shared_files := files;
- lprintf_nl "Exception %s prevents sharing of %s"
- (Printexc2.to_string e) sh.shared_name
+ shared_files := files;
+ lprintf_nl "Exception %s prevents sharing of %s"
+ (Printexc2.to_string e) sh.shared_name
in
job_creater ()
@@ -291,9 +294,9 @@
if !verbose_share then
lprintf_nl "donkeyShare: No info on %s" fullname;
- let found = ref false in
- List.iter (fun sh -> if sh.shared_name = fullname then found := true)
!shared_files;
- if not !found then begin
+ let found = ref false in
+ List.iter (fun sh -> if sh.shared_name = fullname then found := true)
!shared_files;
+ if not !found then begin
let magic =
match Magic.M.magic_fileinfo fullname false with
None -> None
@@ -308,7 +311,7 @@
impl_shared_num = 0;
impl_shared_uploaded = Int64.zero;
impl_shared_ops = pre_shared_ops;
- impl_shared_id = Md4.null;
+ impl_shared_id = Md4.null;
impl_shared_val = pre_shared;
impl_shared_requests = 0;
impl_shared_magic = magic;
@@ -318,12 +321,13 @@
shared_name = fullname;
shared_size = size;
shared_list = [];
- shared_pos = Int64.zero;
+ shared_pos = 0L;
+ shared_chunk = 0;
shared_fd = Unix32.create_ro fullname;
} in
update_shared_num impl;
shared_files := pre_shared :: !shared_files;
- end
+ end
)
let remember_shared_info file new_name =
@@ -340,7 +344,7 @@
with _ ->
if !verbose then
lprintf_nl "Share: Trying mtime on new name %s, disk_name %s,
TODO: too many files in shared_files_new.ini?"
- new_name (file_disk_name file);
+ new_name (file_disk_name file);
Unix32.mtime new_name
in
Index: mldonkey/src/networks/donkey/donkeyTypes.ml
diff -u mldonkey/src/networks/donkey/donkeyTypes.ml:1.40
mldonkey/src/networks/donkey/donkeyTypes.ml:1.41
--- mldonkey/src/networks/donkey/donkeyTypes.ml:1.40 Fri May 19 23:43:54 2006
+++ mldonkey/src/networks/donkey/donkeyTypes.ml Wed May 31 20:02:14 2006
@@ -793,6 +793,7 @@
mutable file_swarmer : CommonSwarming.t option;
mutable file_nchunks : int;
+ mutable file_nchunk_hashes : int;
mutable file_diskname : string;
(* mutable file_chunks : chunk array; *)
(* mutable file_chunks_order : int array; *)
@@ -819,6 +820,7 @@
shared_size : int64;
mutable shared_list : Md4.t list;
mutable shared_pos : int64;
+ mutable shared_chunk : int;
mutable shared_fd : Unix32.t;
shared_shared : file_to_share CommonShared.shared_impl;
}
Index: mldonkey/tools/mld_hash.ml
diff -u mldonkey/tools/mld_hash.ml:1.7 mldonkey/tools/mld_hash.ml:1.8
--- mldonkey/tools/mld_hash.ml:1.7 Thu Feb 2 00:33:28 2006
+++ mldonkey/tools/mld_hash.ml Wed May 31 20:02:14 2006
@@ -131,8 +131,11 @@
(*************************************************************************)
let ed2k_hash_file fd file_size partial =
- let nchunks = Int64.to_int (Int64.pred file_size // block_size) + 1 in
- let md4 = if nchunks = 1 then
+ (* See: DonkeyGlobals *)
+ let nchunks = Int64.to_int (file_size // block_size) + 1 in
+ let nchunk_hashes = Int64.to_int (file_size // block_size) in
+ let nchunk_hashes = if nchunk_hashes <> 0 then nchunk_hashes + 1 else
nchunk_hashes in
+ let md4 = if nchunk_hashes = 0 then
Md4.digest_subfile fd zero file_size
else
let chunks = String.create (nchunks*16) in
@@ -310,16 +313,16 @@
let (sha1, tiger2) = bitprint_file file file_size partial in
lprintf "urn:bitprint:%s.%s\n" (Sha1.to_string sha1)
(TigerTree.to_string tiger2);
- Unix32.close file;
+ Unix32.close file;
lprintf (_b "Renaming...\n");
Unix32.rename file (filename ^ ".final");
lprintf (_b "Removing %s\n") filename;
Unix32.remove file;
let file = f' (filename ^ ".final") file_size in
- Unix32.close file;
+ Unix32.close file;
Unix32.remove file;
- (try Sys.remove "zero_chunk" with _ -> ());
+ (try Sys.remove "zero_chunk" with _ -> ());
lprintf "done\n"
with e ->
- [Mldonkey-commits] mldonkey distrib/ChangeLog src/networks/donkey/..., mldonkey-commits, 2006/05/05
- [Mldonkey-commits] mldonkey distrib/ChangeLog src/networks/donkey/..., mldonkey-commits, 2006/05/05
- [Mldonkey-commits] mldonkey distrib/ChangeLog src/networks/donkey/..., mldonkey-commits, 2006/05/10
- [Mldonkey-commits] mldonkey distrib/ChangeLog src/networks/donkey/..., mldonkey-commits, 2006/05/16
- [Mldonkey-commits] mldonkey distrib/ChangeLog src/networks/donkey/..., mldonkey-commits, 2006/05/19
- [Mldonkey-commits] mldonkey distrib/ChangeLog src/networks/donkey/..., mldonkey-commits, 2006/05/20
- [Mldonkey-commits] mldonkey distrib/ChangeLog src/networks/donkey/..., mldonkey-commits, 2006/05/23
- [Mldonkey-commits] mldonkey distrib/ChangeLog src/networks/donkey/...,
mldonkey-commits <=