--- src/daemon/common/commonTypes.ml.orig 2011-01-23 16:20:26.000000000 +0100 +++ src/daemon/common/commonTypes.ml.orig 2012-03-23 02:47:36.000000000 +0100 @@ -117,7 +117,8 @@ | "md5" -> Md5 (Md5.of_string rem) | "sig2dat" -> Md5Ext (Md5Ext.of_base32 rem) | "bt" | "bittorrent" | "btih" -> - BTUrl (Sha1.of_string rem) + BTUrl (Sha1.of_hexa rem) (* TODO i changed from of_string to of_hexa, because its more common, but both are legal I think. + that is, both base32 and base64 are supposed to work*) | "filetp" -> FileTP (Md4.of_string rem) | _ -> raise (Illegal_urn (s ^ " at " ^ sign ^ " is not known")) --- src/networks/bittorrent/bTClients.ml.orig 2011-07-16 10:59:27.000000000 +0200 +++ src/networks/bittorrent/bTClients.ml.orig 2012-03-23 02:47:36.000000000 +0100 @@ -63,8 +63,11 @@ open BTComplexOptions open BTChooser open BTStats -open TcpMessages + +open TcpMessages +open Bencode +open Str module VB = VerificationBitmap let http_ok = "HTTP 200 OK" @@ -100,6 +103,54 @@ open BTUdpTracker open UdpSocket + +(* some stupid temporary copypasta because i couldnt figure out how to import from btinteractive *) +(* let hack_op_file_cancel file = *) +(* CommonSwarming.remove_swarmer file.file_swarmer; *) +(* file.file_swarmer <- None; *) +(* (\* forward declarations turned out difficult. wtf?*\) *) +(* (\* file_stop file;*\) *) +(* remove_file file; *) +(* (\*disconnect_clients file;*\) *) +(* (\*remove_all_clients file;*\) *) +(* if Sys.file_exists file.file_torrent_diskname then Sys.remove file.file_torrent_diskname *) + +let load_torrent_string s user group = + if !verbose then lprintf_nl "load_torrent_string"; + let file_id, torrent = BTTorrent.decode_torrent s in + + (* Save the torrent, because we later want to put + it in the seeded directory. *) + let torrent_diskname = CommonFile.concat_file downloads_directory (torrent.torrent_name ^ ".torrent") in + if Sys.file_exists torrent_diskname then + begin + if !verbose then lprintf_nl "load_torrent_string: %s already exists, ignoring" torrent_diskname; + raise (Torrent_already_exists torrent.torrent_name) + end; + File.from_string torrent_diskname s; + + if !verbose then + lprintf_nl "Starting torrent download with diskname: %s" + torrent_diskname; + let file = new_download file_id torrent torrent_diskname user group in + (* talk_to_tracker file true; TODO stupid fwd declaration issue. OTOH bep9 torrents arent tracker based anyway*) + CommonInteractive.start_download (file_find (file_num file)); + file + +let load_torrent_file filename user group = + if !verbose then + lprintf_nl "load_torrent_file %s" filename; + let s = File.to_string filename in + (* Delete the torrent if it is in the downloads dir. because it gets saved + again under the torrent name and we don't want to clutter up this dir. .*) + if Sys.file_exists filename + && (Filename.dirname filename) = downloads_directory then + Sys.remove filename; + ignore (load_torrent_string s user group) +(* end copypasta *) + + + let string_of_event = function | READ_DONE -> "READ_DONE" | WRITE_DONE -> "WRITE_DONE" @@ -540,6 +591,7 @@ let reserved () = let s = String.make 8 '\x00' in s.[7] <- (match !bt_dht with None -> '\x00' | Some _ -> '\x01'); + s.[5] <- '\x10'; (* TODO bep9, bep10, notify clients about extended*) s (** handshake *) @@ -569,6 +621,9 @@ *) let send_bitfield c = + if c.client_file.file_metadata_downloading then + lprintf_nl "dont send bitmap, we are in metadata state" + else send_client c (BitField ( match c.client_file.file_swarmer with @@ -607,6 +662,24 @@ c.client_azureus_messaging_protocol <- has_bit 0 0x80 +let send_extended_handshake c file = + let module B = Bencode in + let msg = (B.encode (B.Dictionary [(* "e",B.Int 0L; *) + "m", (B.Dictionary ["ut_metadata", B.Int 1L]); + (* "metadata_size", B.Int (-1L) *)])) in begin + lprintf_file_nl (as_file file) "send extended handshake msg %s" msg; + send_client c (Extended (Int64.to_int 0L, msg)); + end + +let send_extended_piece_request c piece file = + let module B = Bencode in + let msg = (B.encode (B.Dictionary ["msg_type", B.Int 0L; (* 0 is request subtype*) + "piece", B.Int piece; ])) in begin + lprintf_file_nl (as_file file) "send extended request for piece:%Ld msgid:%Ld msg:%s" piece c.client_ut_metadata_msg msg; + send_client c (Extended (Int64.to_int c.client_ut_metadata_msg, msg)); + end + + let show_client c = let (ip,port) = c.client_host in Printf.sprintf "%s:%d %S" (Ip.to_string ip) port (brand_to_string c.client_brand) @@ -705,6 +778,8 @@ begin c.client_incoming <- true; send_init !!client_uid file_id sock; + + send_extended_handshake c file; end; connection_ok c.client_connection_control; if !verbose_msg_clients then @@ -968,7 +1043,7 @@ @param msg The message sent by the client *) and client_to_client c sock msg = - if !verbose_msg_clients then begin + (* if !verbose_msg_clients then begin *) let (ip,port) = (TcpBufferedSocket.peer_addr sock) in let (timeout, next) = get_rtimeout sock in lprintf_nl "CLIENT %d(%s:%d): (%d, %d,%d) Received %s" @@ -977,7 +1052,7 @@ (int_of_float timeout) (int_of_float next) (TcpMessages.to_string msg); - end; + (* end; *) let file = c.client_file in @@ -1079,6 +1154,11 @@ | BitField p -> (*A bitfield is a summary of what a client have*) + lprintf_file_nl (as_file file) "Bitfield message, metadata state %B" c.client_file.file_metadata_downloading ; + if c.client_file.file_metadata_downloading then + lprintf_file_nl (as_file file) "ignoring Bitfield message, we are in metadata state" + else + begin match c.client_file.file_swarmer with None -> () @@ -1125,6 +1205,9 @@ | Have n -> (* A client can send a "Have" without sending a Bitfield *) + if c.client_file.file_metadata_downloading then + lprintf_file_nl (as_file file) "ignoring Have message, we are in metadata state" + else begin match c.client_file.file_swarmer with None -> () @@ -1247,6 +1330,158 @@ if !verbose_msg_clients then lprintf_file_nl (as_file file) "Error: received cancel request but client has no slot" + | Extended (extmsg, payload) -> + + (* extmsg: 0 handshake, N other message previously declared in handshake. + atm ignore extended messages if were not currently in metadata state. + TODO when were not in metadata state we should be friendly and answer metadata requests + *) + if file.file_metadata_downloading then begin + (* since we got at least one extended handshake from the peer, it should be okay to + send a handshake back now. we need to send it so the remote client knows how + to send us messages back. + this should of course be moved but I dont know where yet. + also we shouldnt send more than one handshake of course... + *) + lprintf_file_nl (as_file file) "Got extended msg: %d %s" extmsg payload; + + match extmsg with + 0x0 -> + lprintf_file_nl (as_file file) "Got extended handshake "; + let dict = Bencode.decode payload in begin + match dict with + Dictionary list -> + List.iter (fun (key,value) -> + match key, value with + "metadata_size", Int n -> + lprintf_file_nl (as_file file) "Got metadata size %Ld" n; + c.client_file.file_metadata_size <- n; + | "m", Dictionary mdict -> + lprintf_file_nl (as_file file) "Got meta dict "; + List.iter (fun (key,value) -> + match key, value with + "ut_metadata", Int n -> + lprintf_file_nl (as_file file) "ut_metadata is %Ld " n; + c.client_ut_metadata_msg <- n; + | _ -> (); + ) mdict; + + | _ -> () ; + ) list; + (* okay so now we know what to ask for, so ask for metadata now + since metadata can be larger than 16k which is the limit, the transfer needs to be chunked, so + it is not really right to make the query here. but its a start. + also im just asking for piece 0. + (we should also check that we actually got the metadata info before proceeding) + *) + + + send_extended_handshake c file; + send_extended_piece_request c c.client_file.file_metadata_piece file; + |_ -> () ; + end; + | 0x01 -> (* ut_metadata is 1 because we asked it to be 1 in the handshake + the msg_type is probably + 1 for data, + but could be 0 for request(unlikely since we didnt advertise we had the meta) + 2 for reject, also unlikely since peers shouldnt advertise if they dont have(but will need handling in the end) + + {'msg_type': 1, 'piece': 0, 'total_size': 3425} + after the dict comes the actual piece + + *) + lprintf_file_nl (as_file file) "Got extended ut_metadata message "; + let dict = Bencode.decode payload in + let msgtype = ref 0L in begin + begin + match dict with + Dictionary list -> + List.iter (fun (key,value) -> + match key, value with + "msg_type", Int n -> + lprintf_file_nl (as_file file) "msg_type %Ld" n; + msgtype := n; + | "piece", Int n -> + lprintf_file_nl (as_file file) "piece %Ld" n; + file.file_metadata_piece <- n; + | "total_size", Int n -> + lprintf_file_nl (as_file file) "total_size %Ld" n; (* should always be the same as received in the initial handshake i suppose *) + |_ -> () ; + ) list; + |_ -> () ; + end; + match !msgtype with + 1L -> + let last_piece_index = (Int64.div file.file_metadata_size 16384L) in + lprintf_file_nl (as_file file) "handling metadata piece %Ld of %Ld" + file.file_metadata_piece + last_piece_index; + (* store the metadata piece in memory *) + file.file_metadata_chunks.(1 + (Int64.to_int file.file_metadata_piece)) <- payload; + (* possibly write metadata to disk *) + if file.file_metadata_piece >= + (Int64.div file.file_metadata_size 16384L) then begin + lprintf_file_nl (as_file file) "this was the last piece "; + (* here we should simply delete the current download, and wait for mld to pick up the new torrent file *) + (* the entire payload is currently in the array, TODO *) + let newtorrentfile = (Printf.sprintf "/tmp/BT-%s.torrent" + (Sha1.to_string file.file_id)) in + let fd = Unix32.create_rw newtorrentfile in + let fileindex = ref 0L in + begin + (* the ee is so we can use the same method to find the + start of the payload for the real payloads as well as the synthetic ones + *) + file.file_metadata_chunks.(0) <- "eed4:info"; + file.file_metadata_chunks.(2 + Int64.to_int last_piece_index) <- "eee"; + try + Array.iteri (fun index chunk -> + (* regexp ee is a fugly way to find the end of the 1st dict before the real payload *) + let metaindex = (2 + (Str.search_forward (Str.regexp_string "ee") chunk 0 )) in + let chunklength = ((String.length chunk) - metaindex) in + Unix32.write fd !fileindex chunk + metaindex + chunklength; + fileindex := Int64.add !fileindex (Int64.of_int chunklength); + (); + ) file.file_metadata_chunks; + with e -> begin + (* TODO ignoring errors for now, the array isnt really set up right anyway yet *) + lprintf_file_nl (as_file file) "Error %s saving metadata" + (Printexc2.to_string e) + end; + (* Yay, now the new torrent is on disk! amazing! However, now we need to kill the dummy torrent + and restart it with the fresh real torrent *) + + (* it seems we need to use the dynamic interface... *) + lprintf_file_nl (as_file file) "cancelling metadata download "; + let owner = file.file_file.impl_file_owner in + let group = file.file_file.impl_file_group in begin + CommonInteractive.file_cancel (as_file file) owner ; + (* hack_op_file_cancel c.client_file; *) + lprintf_file_nl (as_file file) "starting download from metadata torrent %s" newtorrentfile ; + load_torrent_file newtorrentfile owner group; + end; + end; + + end + else begin + (* now ask for the next metadata piece, if any *) + let module B = Bencode in + let nextpiece = (Int64.succ file.file_metadata_piece) in begin + lprintf_file_nl (as_file file) "asking for the next piece %Ld" nextpiece; + send_extended_piece_request c nextpiece file; + end; + end; + |_ -> + lprintf_file_nl (as_file file) "unmatched extended subtype" ; + end; + + + | _ -> + lprintf_file_nl (as_file file) "Got extended other msg "; + end; + | DHT_Port port -> match !bt_dht with | None -> @@ -1261,6 +1496,7 @@ BT_DHT.update dht Kademlia.Good id addr end + with e -> lprintf_file_nl (as_file file) "Error %s while handling MESSAGE: %s" (Printexc2.to_string e) (TcpMessages.to_string msg) @@ -1332,6 +1568,8 @@ lprintf_file_nl (as_file file) "READY TO DOWNLOAD FILE"; send_init !!client_uid file.file_id sock; + send_extended_handshake c file; + (* Fabrice: Initialize the client bitmap and uploader fields to <> None *) update_client_bitmap c; (* (try get_from_client sock c with _ -> ());*) --- src/networks/bittorrent/bTGlobals.ml.orig 2011-10-25 19:33:21.000000000 +0200 +++ src/networks/bittorrent/bTGlobals.ml.orig 2012-03-23 02:47:36.000000000 +0100 @@ -272,7 +272,7 @@ file.file_trackers <- t :: file.file_trackers) file_trackers -let new_file file_id t torrent_diskname file_temp file_state user group = +let new_file ?(metadata = false) file_id t torrent_diskname file_temp file_state user group = try Hashtbl.find files_by_uid file_id with Not_found -> @@ -302,6 +302,10 @@ file_session_uploaded = Int64.zero; file_session_downloaded = Int64.zero; file_last_dht_announce = 0; + file_metadata_size = 0L; + file_metadata_piece = 0L; + file_metadata_downloading = metadata; + file_metadata_chunks = Array.make 20 ""; file_private = t.torrent_private; } and file_impl = { (dummy_file_impl ()) with @@ -354,10 +358,10 @@ must_share_file file; file -let new_download file_id t torrent_diskname user = +let new_download ?(metadata = false) file_id t torrent_diskname user = let file_temp = Filename.concat !!DO.temp_directory (Printf.sprintf "BT-%s" (Sha1.to_string file_id)) in - new_file file_id t torrent_diskname file_temp FileDownloading user + new_file ~metadata:metadata file_id t torrent_diskname file_temp FileDownloading user let ft_by_num = Hashtbl.create 13 let ft_counter = ref 0 @@ -866,6 +870,7 @@ client_cache_extension = false; client_fast_extension = false; client_utorrent_extension = false; + client_ut_metadata_msg = -1L; client_azureus_messaging_protocol = false; } and impl = { dummy_client_impl with --- src/networks/bittorrent/bTInteractive.ml.orig 2011-07-16 10:59:27.000000000 +0200 +++ src/networks/bittorrent/bTInteractive.ml.orig 2012-03-23 02:47:36.000000000 +0100 @@ -322,7 +322,8 @@ if !bt_dht <> None then emit (_s"Last DHT announce") ~desc:(_s"Last time this torrent was announced in DHT") (string_of_date file.file_last_dht_announce); - + emit (_s"Metadata downloading") (if file.file_metadata_downloading then _s "yes" else _s "no"); + let rec print_first_tracker l = match l with | [] -> () @@ -837,7 +838,8 @@ List.iter (fun file -> (* if !verbose_share then lprintf_nl "Checking torrent share for %s" file.file_torrent_diskname; *) if not (Sys.file_exists file.file_torrent_diskname) && - file_state file = FileShared then + file_state file = FileShared && + not (file.file_metadata_downloading ) then begin if !verbose_share then lprintf_nl "Removing torrent share for %s" file.file_torrent_diskname; BTClients.file_stop file; @@ -1251,6 +1253,72 @@ _s "" ), " :\t\t\tstart BT download"; + "startbthash", "Network/Bittorrent", Arg_one (fun hashstr o -> + begin + let torrent = { + torrent_name = hashstr; + torrent_filename = hashstr; + torrent_name_utf8 = hashstr; + torrent_comment = ""; + torrent_pieces = Array.of_list []; + torrent_piece_size = 1L; + torrent_files = []; + torrent_length = 1L; + torrent_created_by = ""; + torrent_creation_date = 1000000L; + torrent_modified_by = ""; + torrent_encoding = ""; + torrent_private = false; + torrent_announce = ""; + torrent_announce_list = []; + } in + let hash = (Sha1.of_hexa hashstr) in + new_download ~metadata:true hash torrent "" o.conn_user.ui_user o.conn_user.ui_user.user_default_group; + (); + + end; + _s "" + ), " :\t\t\tstart BT download using a hash DEBUG will go away"; + + "startbtmagnet", "Network/Bittorrent", Arg_one (fun magneturl o -> + begin + let exn_catch f x = try `Ok (f x) with exn -> `Exn exn in + match exn_catch parse_magnet_url magneturl with + | `Exn _ -> "Not a magnet url", false + | `Ok magnet -> + if !verbose then + lprintf_nl "Got magnet url %S" magneturl; + List.iter (fun(v) -> lprintf_nl "magnet %s" (string_of_uid v)) magnet#uids ; + match List2.filter_map (function BTUrl btih -> Some btih | _ -> None) magnet#uids with + | [] -> "No btih found in magnet url", false; + | btih::_ -> + lprintf_nl "Got btih %S" (Sha1.to_string btih); + let hashstr = (Sha1.to_string btih) in + let torrent = { + torrent_name = hashstr; (*magnet#name*) + torrent_filename = hashstr; + torrent_name_utf8 = hashstr; + torrent_comment = ""; + torrent_pieces = Array.of_list []; + torrent_piece_size = 1L; + torrent_files = []; + torrent_length = 1L; + torrent_created_by = ""; + torrent_creation_date = 1000000L; + torrent_modified_by = ""; + torrent_encoding = ""; + torrent_private = false; + torrent_announce = ""; + torrent_announce_list = []; + } in + new_download ~metadata:true btih torrent "" o.conn_user.ui_user o.conn_user.ui_user.user_default_group; + magnet#name, true; + end; + _s "" + ), " :\t\t\tstart BT download using a bt magnet url DEBUG will go away"; + + + "stop_all_bt", "Network/Bittorrent", Arg_none (fun o -> List.iter (fun file -> BTClients.file_stop file ) !current_files; let buf = o.conn_buf in --- src/networks/bittorrent/bTProtocol.ml.orig 2011-01-23 16:20:26.000000000 +0100 +++ src/networks/bittorrent/bTProtocol.ml.orig 2012-03-23 02:47:36.000000000 +0100 @@ -217,10 +217,11 @@ * 9 - DHT port announcement int16: UDP port + * 20 - extended TODO Choke/unchoke every 10 seconds *) - +open String open BasicSocket open CommonTypes open Printf2 @@ -251,6 +252,8 @@ mutable gconn_close_on_write : bool; } + + module TcpMessages = struct type msg = @@ -266,9 +269,11 @@ | Ping | PeerID of string | DHT_Port of int - + | Extended of int * string + let to_string msg = - match msg with + match + msg with | Choke -> "Choke" | Unchoke -> "Unchoke" | Interested -> "Interested" @@ -284,6 +289,7 @@ | Ping -> "Ping" | PeerID s -> Printf.sprintf "PeerID [%s]" (String.escaped s) | DHT_Port n -> Printf.sprintf "DHT_Port %d" n + | Extended (n, s) -> Printf.sprintf "Extended [%d %s]" n (String.escaped s) let parsing opcode m = match opcode with @@ -297,6 +303,7 @@ | 7 -> Piece (get_int m 0, get_uint64_32 m 4, m, 8, String.length m - 8) | 8 -> Cancel (get_int m 0, get_uint64_32 m 4, get_uint64_32 m 8) | 9 -> DHT_Port (get_int16 m 0) + | 20 -> Extended (get_int8 m 0, (String.sub m 1 ((String.length m) - 1))) | -1 -> PeerID m | _ -> raise Not_found @@ -325,6 +332,7 @@ | PeerID _ -> () | Ping -> () | DHT_Port n -> buf_int8 buf 9; buf_int16 buf n + | Extended (n, string) -> buf_int8 buf 20; buf_int8 buf n; Buffer.add_string buf string end; let s = Buffer.contents buf in str_int s 0 (String.length s - 4); --- src/networks/bittorrent/bTTypes.ml.orig 2011-01-23 16:20:26.000000000 +0100 +++ src/networks/bittorrent/bTTypes.ml.orig 2012-03-23 02:47:36.000000000 +0100 @@ -287,6 +287,9 @@ mutable client_cache_extension : bool; mutable client_fast_extension : bool; mutable client_utorrent_extension : bool; + + mutable client_ut_metadata_msg : int64; + mutable client_azureus_messaging_protocol : bool; } @@ -335,6 +338,12 @@ mutable file_session_downloaded : int64; (** DHT specific *) mutable file_last_dht_announce : int; + + mutable file_metadata_size : int64; + mutable file_metadata_piece : int64; + mutable file_metadata_downloading : bool; + mutable file_metadata_chunks : string array; + file_private : bool; }