# # add_file "globish.cc" # # add_file "globish.hh" # # add_file "hmac.cc" # # add_file "hmac.hh" # # add_file "tests/t_netsync_globs.at" # # patch "AUTHORS" # from [30e965bc57edd221eb6aa30839a20299ab66991f] # to [7a3afb6e49bc3428635204b2eed5f39c7375c950] # # patch "ChangeLog" # from [c357fc0eef58e7efb9a987ae2be4aa799261bf3f] # to [acc813649d4cffb7939260314cc6d26ae1f7f789] # # patch "Makefile.am" # from [bbaa41aa626f9b9c3875ad2f6f9888f018c37e84] # to [ca602dd61d7c6f3c4a1b290c56ae7ac1a67f0451] # # patch "commands.cc" # from [aaf375728369d96c9d35d37d6feea20bb97f6c8f] # to [d155ea587e6ee57f7057e497579cf08a4882fb8c] # # patch "constants.cc" # from [eabc0d5838a67e3e4d351ce3d28eac0387f773ae] # to [dcbffe20599c26310957ee7c1310a44ae93dc2a0] # # patch "constants.hh" # from [4a4494bcaa97ec07c147fc066564519bb4d13481] # to [94b12534ecf00b0d10056ab3edf9fad2e38eb300] # # patch "globish.cc" # from [] # to [276e3e515b8fbdc98b00b49c6f7f331affd0053f] # # patch "globish.hh" # from [] # to [bcbaf2efddd720efb786f5f2843890071e42e3d8] # # patch "hmac.cc" # from [] # to [b3ff5a538eb8986fddf018c89f2352abc45e73dd] # # patch "hmac.hh" # from [] # to [1f62c61985e4369d1ee5a29c5b8e3525bc03ebfe] # # patch "keys.cc" # from [a4ccef56d24ee9cda7a009144f5476e3cdcb07d7] # to [9c50dc0d669ffb6ad2328c46485bf3ae65dad4b5] # # patch "keys.hh" # from [f10aaea8dbaf4005a55ec388f278d1bb81d664af] # to [ec2da9cbde80341456204b6e0e0b4f727388ed8b] # # patch "lua.cc" # from [e4dd123e0c07b59361555b521f3b2cc17aa445b5] # to [a8af56ad7fe0d9baa85c265b05dd94877a806c19] # # patch "lua.hh" # from [0c4357742dc32474757a7905380b407e532563f4] # to [0c4f6966a60230b591f07d0a0519485bf451a349] # # patch "monotone.1" # from [8b3699fb3547458585d9e9acb48f4baf5fd0b6fa] # to [da6b32b820606a2fa282d47374330c0ca455ffb8] # # patch "monotone.texi" # from [8c87d78f0fd385725bd9cec646ddbddae29b9a34] # to [aea7f0c2241348081d1c31ab77cdab0cbe18d6fe] # # patch "netcmd.cc" # from [9b3a263d43423df25f0da8714741f590d4d2f41a] # to [844f0f7559a6415ef30a28d99fa6ea76a1c71a0c] # # patch "netcmd.hh" # from [5626696805eab24ff5bd23f114a749b3aa78645a] # to [f6262bdd5719defdd5ef6237e090687701ef4006] # # patch "netsync.cc" # from [3251de2cb2bc135ba3576d742cbe7ae2cd5a30e2] # to [e9786aaaad7990675f952101b544f35fd168df4c] # # patch "netsync.hh" # from [758b6ef261dc3d69e99e4fbcd79571e89749c9f4] # to [276fdfd34e38b36adc1a45d012792fd43db07a2c] # # patch "tests/t_netsync_defaults.at" # from [c041c4c1f93c04041b0fb53257e56c4b3b24a2cf] # to [ad699c5f7470abec2b97ac1b4d0f71250d134d25] # # patch "tests/t_netsync_globs.at" # from [] # to [3dff1121e0e6897d4f16ed9141253806e5b65939] # # patch "tests/t_netsync_permissions.at" # from [73b0366351e254dd0c272fc9d01c04d2b5fba795] # to [fb766246587fa8cf19f1b2952ec63be65ad82b63] # # patch "tests/t_netsync_single.at" # from [10e65a3f83698c36866333909f01cbdb6915efe3] # to [b3d48b1881c63ccb549d37726ae2d7144a856820] # # patch "testsuite.at" # from [828c3a9002709b8d6e7bb0c3d4e1e0cfdd69990d] # to [4cbb082cf10749231aaffa05b12dfdd51e8e2096] # # patch "transforms.cc" # from [041db043a6e340a55705de868e77a0fad8b5a413] # to [589f2e927da5c33a2adb9bfea8f9b70d041ce212] # # patch "transforms.hh" # from [beb9c11a04e735f442928aea2e984834c8573c8d] # to [9f9adf9a36619e493cceea57d78e6c4cc456598d] # # patch "unit_tests.cc" # from [230b4227d08bed323a2625f593073427bca9fe9b] # to [99e4b00f7e1071461ac6d79372edda008bb5b182] # # patch "unit_tests.hh" # from [b4b20ac190077a08db547eb1dcd79f40c58b2ebc] # to [340f8c9e449facd6744cf6b926367fbd98bbb062] # # patch "vocab.cc" # from [6e07baffa263c80f013d474ad362c3b52c162201] # to [d76e379b5a6bc88ed11268a9a63cd4f4ecc798c0] # # patch "vocab_terms.hh" # from [c12dd87ed775d3e2d8713a393cc055ce68e5ae16] # to [fe3bf7a7700bdde82211ba0be39caf248025e00b] # --- AUTHORS +++ AUTHORS @@ -60,6 +60,8 @@ Matthew Gregan Riccardo Ghetta Brian Campbell + Ethan Blanton + Eric Anderson supporting files: ----------------- --- ChangeLog +++ ChangeLog @@ -140,6 +140,210 @@ in $EDITOR, otherwise use emacs. * revision.cc (check_sane_history): Remove stale comment. +2005-07-05 Nathaniel Smith + + * globish.cc (combine_and_check_globish): Don't add unnecessary + {}'s. + * tests/t_netsync_globs.at, testsuite.at: New test. + +2005-07-04 Nathaniel Smith + + * netcmd.cc (do_netcmd_roundtrip, test_netcmd_mac): Update for new + chained_hmac object. + * constants.hh (netsync_key_initializer): Update comment. + * hmac.hh (hmac_length): Expose length of MACs. + * hmac.cc: I() that it matches what CryptoPP wants to give. + * netcmd.cc: I() that it matches the length hard-coded into the + netsync protocol. + * vocab.cc (verify(netsync_hmac_value)): Fix error message. + +2005-07-04 Nathaniel Smith + + * tests/t_netsync_defaults.at: Update for new var names. All + tests now pass. + +2005-07-04 Nathaniel Smith + + * lua.cc (hook_get_netsync_write_permitted): Fix typo. + +2005-07-04 Nathaniel Smith + + * globish.cc (globish_matcher_test): Add check for {foo} (no + commas). + +2005-07-04 Nathaniel Smith + + * globish.cc (checked_globish_to_regex): Make the special case for + the empty pattern, actually work. Unit tests now pass. + +2005-07-04 Nathaniel Smith + + * netcmd.cc (test_netcmd_functions): Update for new anonymous/auth + packet formats. + +2005-07-04 Nathaniel Smith + + * monotone.texi, monotone.1: Update for new glob stuff. + * commands.cc (process_netsync_args, push, pull, sync, serve): + 'serve' always requires arguments, rather than falling back on db + defaults. + +2005-07-04 Nathaniel Smith + + * commands.cc (process_netsync_args, push, pull, sync, serve): + Adapt for patterns instead of regexen; slight refactoring too. + +2005-07-03 Nathaniel Smith + + * netsync.cc: Finally self-consistent. + +2005-07-03 Nathaniel Smith + + * netsync.hh (run_netsync_protocol): Fix prototype. + +2005-07-03 Nathaniel Smith + + * globish.hh: Document the empty pattern as never matching. + * globish.cc (checked_globish_to_regex): Implement it. + (globish_matcher_test): Check it. + +2005-07-03 Nathaniel Smith + + * monotone.texi (Network Service, Hooks): + * testsuite.at: + * tests/t_netsync_permissions.at: + * tests/t_netsync_single.at: Update to match new + get_netsync_write_permitted definition. + +2005-07-03 Nathaniel Smith + + * lua.{cc,hh} (hook_get_netsync_write_permitted): Don't take a + branch argument; write permission is now all or none. (It really + was before anyway...) + * netsync.cc: Update accordingly. + +2005-07-03 Nathaniel Smith + + * netsync.cc: More updating for pattern stuff; getting there... + +2005-06-28 Nathaniel Smith + + * netsync.cc: Update low-level functions to use include_pattern + and exclude_pattern. + +2005-06-28 Nathaniel Smith + + * netcmd.{cc,hh} (read_anonymous_cmd, write_anonymous_cmd) + (read_auth_cmd, write_auth_cmd): Take include_pattern and + exclude_pattern arguments. + +2005-06-28 Nathaniel Smith + + * globish.{cc,hh}: New files. + * Makefile.am (MOST_SOURCES): Add them. + * transforms.{cc,hh}: Remove glob-related stuff. + * unit_tests.{cc,hh}: Call globish unit tests. + +2005-06-27 Nathaniel Smith + + * transforms.cc (glob_to_regex, globs_to_regex, regexes_to_regex): + Choose "regex" as standard spelling. Clean up code, add code for + handling sets, start improving tests (don't currently pass). + * transforms.hh (glob_to_regex, globs_to_regex, regexes_to_regex): + Prototype. + +2005-06-28 Matt Johnston + + * constants.cc: increase db_version_cache_sz to 7 MB + * netsync.cc: use a deque rather than a single + string buffer for outbuf. + * netsync.cc (arm): only queue data when there is + available space + * AUTHORS: added Eric Anderson + +2005-06-26 Matt Johnston + + * transforms.hh: remove extraneous #ifdef + * hmac.cc, hmac.hh: actually add them + +2005-06-26 Matt Johnston + + * netcmd.cc (netcmd::read, netcmd::write): change to using a HMACs + chained by including the previous HMAC in the input data, rather + than altering the key each time. + * netcmd.cc ({read,write}_{data,delta}_cmd): use encode_gzip/decode_gzip + rather than raw xform. + * hmac.{cc,hh}: new chained_hmac abstraction + * Makefile.in: add them + * netsync.cc: each session keeps a chained_hmac for read/write + * transforms.hh: add a string variant for encode_gzip + +2005-06-25 Nathaniel Smith + + * netsync.cc: Tweak comment. + +2005-06-25 Nathaniel Smith + + * AUTHORS: Add Ethan Blanton . + +2005-06-22 Nathaniel Smith + + * netcmd.hh (netcmd::read, netcmd::write): Don't have defaults for + key/hmac arguments. + * netcmd.cc (do_netcmd_roundtrip): New function. + (test_netcmd_functions): Use it. Also, make work with hmac + changes. + (test_netcmd_mac): New test. + (add_netcmd_tests): Call it. + +2005-06-22 Nathaniel Smith + + * netcmd.cc (read): Remove unused variable. + * netsync.cc (call_server, process) + (arm_sessions_and_calculate_probe, handle_read_available): Give + better error message on bad_decode exceptions. + +2005-06-22 Nathaniel Smith + + * netcmd.cc, netsync.cc: Revert backwards compatibility code; 0.19 + and 0.20 can't be usefully compatible, and the code as it existed + would cause real version mismatch error reporting to not work + right. (Old client with new server would give a generic "server + disconnected" error message instead of something useful.) + +2005-06-21 Nathaniel Smith + + * netsync.cc (rebuild_merkle_trees): Fix FIXME comments to match + reality. + * tests/t_netsync_diffbranch.at: No longer a bug, remove + priority. + +2005-06-20 Nathaniel Smith + + * monotone.texi (Hook Reference): Oops, missed a @ref. + +2005-06-20 Nathaniel Smith + + * monotone.texi (Default monotonerc): Rename section to... + (Default hooks): ...this, to emphasize is still read even when a + monotonerc exists. + +2005-06-19 Richard Levitte + + * Makefile.am: There's no reason for monotone.pdf or .dvi to + depend on monotone.info, since they are built from the .texi + files. Also, make the monotone.html and html targets depend + on version.texi and std_hooks.lua as well. + +2005-06-18 Matt Johnston + + * INSTALL: fix typo, should be -Iboost_1_31_0 not -Iboost_1_31_2 + +2005-06-18 Riccardo Ghetta + * monotone.texi: include std_hooks.lua as an appendix and remove long + lua excerpts from hook reference. + * Makefile.am : make monotone.pdf/eps depend on monotone.info + 2005-06-24 Matt Johnston * transforms.{cc,hh}: combine gzip and base64 in one --- Makefile.am +++ Makefile.am @@ -40,6 +40,8 @@ selectors.cc selectors.hh \ annotate.cc annotate.hh \ restrictions.cc restrictions.hh \ + hmac.cc hmac.hh \ + globish.cc globish.hh \ \ cleanup.hh unit_tests.hh interner.hh \ cycle_detector.hh randomfile.hh adler32.hh quick_alloc.hh \ --- commands.cc +++ commands.cc @@ -49,6 +49,7 @@ #include "selectors.hh" #include "annotate.hh" #include "options.hh" +#include "globish.hh" // // this file defines the task-oriented "top level" commands which can be @@ -1976,22 +1977,23 @@ static const var_key default_server_key(var_domain("database"), var_name("default-server")); -static const var_key default_pattern_key(var_domain("database"), - var_name("default-pattern")); +static const var_key default_include_pattern_key(var_domain("database"), + var_name("default-include-pattern")); +static const var_key default_exclude_pattern_key(var_domain("database"), + var_name("default-exclude-pattern")); static void -process_netsync_client_args(std::string const & name, - std::vector const & args, - utf8 & addr, std::vector & patterns, - app_state & app) +process_netsync_args(std::string const & name, + std::vector const & args, + utf8 & addr, + utf8 & include_pattern, utf8 & exclude_pattern, + bool use_defaults, + app_state & app) { - if (args.size() > 2) - throw usage(name); - if (args.size() >= 1) { addr = idx(args, 0); - if (!app.db.var_exists(default_server_key)) + if (use_defaults && !app.db.var_exists(default_server_key)) { P(F("setting default server to %s\n") % addr); app.db.set_var(default_server_key, var_value(addr())); @@ -1999,6 +2001,7 @@ } else { + N(use_defaults, F("no hostname given")); N(app.db.var_exists(default_server_key), F("no server given and no default server set")); var_value addr_value; @@ -2006,74 +2009,73 @@ addr = utf8(addr_value()); L(F("using default server address: %s\n") % addr); } - // NB: even though the netsync code wants a vector of patterns, in fact - // this only works for the server; when we're a client, our vector must have - // length exactly 1. - utf8 pattern; if (args.size() >= 2) { - pattern = idx(args, 1); - if (!app.db.var_exists(default_pattern_key)) + std::set patterns(args.begin() + 1, args.end()); + combine_and_check_globish(patterns, include_pattern); + if (use_defaults && !app.db.var_exists(default_include_pattern_key)) { - P(F("setting default regex pattern to %s\n") % pattern); - app.db.set_var(default_pattern_key, var_value(pattern())); + P(F("setting default branch pattern to %s\n") % include_pattern); + app.db.set_var(default_include_pattern_key, var_value(include_pattern())); } } else { - N(app.db.var_exists(default_pattern_key), - F("no regex pattern given and no default pattern set")); + N(use_defaults, F("no branch pattern given")); + N(app.db.var_exists(default_include_pattern_key), + F("no branch pattern given and no default pattern set")); var_value pattern_value; - app.db.get_var(default_pattern_key, pattern_value); - pattern = utf8(pattern_value()); - L(F("using default regex pattern: %s\n") % pattern); + app.db.get_var(default_include_pattern_key, pattern_value); + include_pattern = utf8(pattern_value()); + L(F("using default branch pattern: %s\n") % include_pattern); } - patterns.push_back(pattern); + + // For now, don't handle excludes. + exclude_pattern = utf8(""); } -CMD(push, "network", "[ADDRESS[:PORTNUMBER] [REGEX]]", +CMD(push, "network", "[ADDRESS[:PORTNUMBER] [PATTERN]]", "push branches matching REGEX to netsync server at ADDRESS", OPT_NONE) { - utf8 addr; - vector patterns; - process_netsync_client_args(name, args, addr, patterns, app); + utf8 addr, include_pattern, exclude_pattern; + process_netsync_args(name, args, addr, include_pattern, exclude_pattern, true, app); rsa_keypair_id key; N(guess_default_key(key, app), F("could not guess default signing key")); app.signing_key = key; - run_netsync_protocol(client_voice, source_role, addr, patterns, app); + run_netsync_protocol(client_voice, source_role, addr, + include_pattern, exclude_pattern, app); } -CMD(pull, "network", "[ADDRESS[:PORTNUMBER] [REGEX]]", +CMD(pull, "network", "[ADDRESS[:PORTNUMBER] [PATTERN]]", "pull branches matching REGEX from netsync server at ADDRESS", OPT_NONE) { - utf8 addr; - vector patterns; - process_netsync_client_args(name, args, addr, patterns, app); + utf8 addr, include_pattern, exclude_pattern; + process_netsync_args(name, args, addr, include_pattern, exclude_pattern, true, app); if (app.signing_key() == "") W(F("doing anonymous pull\n")); - run_netsync_protocol(client_voice, sink_role, addr, patterns, app); + run_netsync_protocol(client_voice, sink_role, addr, + include_pattern, exclude_pattern, app); } -CMD(sync, "network", "[ADDRESS[:PORTNUMBER] [REGEX]]", +CMD(sync, "network", "[ADDRESS[:PORTNUMBER] [PATTERN]]", "sync branches matching REGEX with netsync server at ADDRESS", OPT_NONE) { - utf8 addr; - vector patterns; - process_netsync_client_args(name, args, addr, patterns, app); + utf8 addr, include_pattern, exclude_pattern; + process_netsync_args(name, args, addr, include_pattern, exclude_pattern, true, app); rsa_keypair_id key; N(guess_default_key(key, app), F("could not guess default signing key")); app.signing_key = key; - run_netsync_protocol(client_voice, source_and_sink_role, addr, patterns, - app); + run_netsync_protocol(client_voice, source_and_sink_role, addr, + include_pattern, exclude_pattern, app); } -CMD(serve, "network", "ADDRESS[:PORTNUMBER] REGEX ...", +CMD(serve, "network", "ADDRESS[:PORTNUMBER] PATTERN ...", "listen on ADDRESS and serve the specified branches to connecting clients", OPT_PIDFILE) { if (args.size() < 2) @@ -2089,9 +2091,10 @@ F("need permission to store persistent passphrase (see hook persist_phrase_ok())")); require_password(key, app); - utf8 addr(idx(args,0)); - vector patterns(args.begin() + 1, args.end()); - run_netsync_protocol(server_voice, source_and_sink_role, addr, patterns, app); + utf8 addr, include_pattern, exclude_pattern; + process_netsync_args(name, args, addr, include_pattern, exclude_pattern, false, app); + run_netsync_protocol(server_voice, source_and_sink_role, addr, + include_pattern, exclude_pattern, app); } --- constants.cc +++ constants.cc @@ -44,8 +44,11 @@ // truncated. size_t const db_log_line_sz = 70; - // size in bytes of the database xdelta version reconstruction cache - size_t const db_version_cache_sz = 1 << 20; + // size in bytes of the database xdelta version reconstruction cache. + // the value of 7 MB was determined as the optimal point after timing + // various values with a pull of the monotone repository - it could + // be tweaked further. + size_t const db_version_cache_sz = 7 * (1 << 20); // size of a line of text in the log buffer, beyond which log lines will be // truncated. @@ -160,5 +163,9 @@ size_t const netsync_default_port = 5253; size_t const netsync_connection_limit = 1024; size_t const netsync_timeout_seconds = 21600; // 6 hours + size_t const netsync_session_key_length_in_bytes = 20; // 160 bits + size_t const netsync_hmac_value_length_in_bytes = 20; // 160 bits + std::string const & netsync_key_initializer = std::string(netsync_session_key_length_in_bytes, 0); + } --- constants.hh +++ constants.hh @@ -7,6 +7,7 @@ // see the file COPYING for details #include +#include #include "numeric_vocab.hh" namespace constants @@ -120,6 +121,15 @@ // number of seconds a connection can be idle before it's dropped extern size_t const netsync_timeout_seconds; + // netsync HMAC key length + extern size_t const netsync_session_key_length_in_bytes; + + // netsync HMAC value length + extern size_t const netsync_hmac_value_length_in_bytes; + + // netsync session key default initializer + extern std::string const & netsync_key_initializer; + } #endif // __CONSTANTS_HH__ --- globish.cc +++ globish.cc @@ -0,0 +1,229 @@ +// copyright (C) 2005 Richard Levitte +// copyright (C) 2005 nathaniel smith +// all rights reserved. +// licensed to the public under the terms of the GNU GPL (>= 2) +// see the file COPYING for details + +#include "sanity.hh" +#include "globish.hh" + +// this converts a globish pattern to a regex. The regex should be usable by +// the Boost regex library operating in default mode, i.e., it should be a +// valid ECMAscript regex. +// +// Pattern tranformation: +// +// - As a special case, the empty pattern is translated to "$.^", which cannot +// match any string. +// +// - Any character except those described below are copied as they are. +// - The backslash (\) escapes the following character. The escaping +// backslash is copied to the regex along with the following character. +// - * is transformed to .* in the regex. +// - ? is transformed to . in the regex. +// - { is transformed to ( in the regex +// - } is transformed to ) in the regex +// - , is transformed to | in the regex, if within { and } +// - ^ is escaped unless it comes directly after an unescaped [. +// - ! is transformed to ^ in the regex if it comes directly after an +// unescaped [. +// - ] directly following an unescaped [ is escaped. +static void +maybe_quote(char c, std::string & re) +{ + if (!(isalnum(c) || c == '_')) + { + re += '\\'; + } + re += c; +} + +static void +checked_globish_to_regex(std::string const & glob, std::string & regex) +{ + int in_braces = 0; // counter for levels if {} + + regex.clear(); + regex.reserve(glob.size() * 2); + + L(F("checked_globish_to_regex: input = '%s'\n") % glob); + + if (glob == "") + { + regex = "$.^"; + // and the below loop will do nothing + } + for (std::string::const_iterator i = glob.begin(); i != glob.end(); ++i) + { + char c = *i; + + N(in_braces < 5, F("braces nested too deep in pattern '%s'") % glob); + + switch(c) + { + case '*': + regex += ".*"; + break; + case '?': + regex += '.'; + break; + case '{': + in_braces++; + regex += '('; + break; + case '}': + N(in_braces != 0, + F("trying to end a brace expression in a glob when none is started")); + regex += ')'; + in_braces--; + break; + case ',': + if (in_braces > 0) + regex += '|'; + else + maybe_quote(c, regex); + break; + case '\\': + N(++i != glob.end(), F("pattern '%s' ends with backslash") % glob); + maybe_quote(*i, regex); + break; + default: + maybe_quote(c, regex); + break; + } + } + + N(in_braces == 0, + F("run-away brace expression in pattern '%s'") % glob); + + L(F("checked_globish_to_regex: output = '%s'\n") % regex); +} + +void +combine_and_check_globish(std::set const & patterns, utf8 & pattern) +{ + std::string p; + if (patterns.size() > 1) + p += '{'; + bool first = true; + for (std::set::const_iterator i = patterns.begin(); i != patterns.end(); ++i) + { + std::string tmp; + // run for the checking it does + checked_globish_to_regex((*i)(), tmp); + if (!first) + p += ','; + first = false; + p += (*i)(); + } + if (patterns.size() > 1) + p += '}'; + pattern = utf8(p); +} + +globish_matcher::globish_matcher(utf8 const & include_pat, utf8 const & exclude_pat) +{ + std::string re; + checked_globish_to_regex(include_pat(), re); + r_inc = re; + checked_globish_to_regex(exclude_pat(), re); + r_exc = re; +} + +bool +globish_matcher::operator()(std::string const & s) +{ + // regex_match may throw a std::runtime_error, if the regex turns out to be + // really pathological + return boost::regex_match(s, r_inc) && !boost::regex_match(s, r_exc); +} + +#ifdef BUILD_UNIT_TESTS +#include "unit_tests.hh" + +static void +checked_globish_to_regex_test() +{ + std::string pat; + + checked_globish_to_regex("*", pat); + BOOST_CHECK(pat == ".*"); + checked_globish_to_regex("?", pat); + BOOST_CHECK(pat == "."); + checked_globish_to_regex("{a,b,c}d", pat); + BOOST_CHECK(pat == "(a|b|c)d"); + checked_globish_to_regex("foo{a,{b,c},?*}d", pat); + BOOST_CHECK(pat == "foo(a|(b|c)|..*)d"); + checked_globish_to_regex("\\a\\b\\|\\{\\*", pat); + BOOST_CHECK(pat == "ab\\|\\{\\*"); + checked_globish_to_regex(".+$^{}", pat); + BOOST_CHECK(pat == "\\.\\+\\$\\^()"); + checked_globish_to_regex(",", pat); + // we're very conservative about metacharacters, and quote all + // non-alphanumerics, hence the backslash + BOOST_CHECK(pat == "\\,"); + + BOOST_CHECK_THROW(checked_globish_to_regex("foo\\", pat), informative_failure); + BOOST_CHECK_THROW(checked_globish_to_regex("{foo", pat), informative_failure); + BOOST_CHECK_THROW(checked_globish_to_regex("{foo,bar{baz,quux}", pat), informative_failure); + BOOST_CHECK_THROW(checked_globish_to_regex("foo}", pat), informative_failure); + BOOST_CHECK_THROW(checked_globish_to_regex("foo,bar{baz,quux}}", pat), informative_failure); + BOOST_CHECK_THROW(checked_globish_to_regex("{{{{{{{{{{a,b},c},d},e},f},g},h},i},j},k}", pat), informative_failure); +} + +static void +combine_and_check_globish_test() +{ + std::set s; + s.insert(utf8("a")); + s.insert(utf8("b")); + s.insert(utf8("c")); + utf8 combined; + combine_and_check_globish(s, combined); + BOOST_CHECK(combined() == "{a,b,c}"); +} + +static void +globish_matcher_test() +{ + { + globish_matcher m(utf8("{a,b}?*\\*|"), utf8("*c*")); + BOOST_CHECK(m("aq*|")); + BOOST_CHECK(m("bq*|")); + BOOST_CHECK(!m("bc*|")); + BOOST_CHECK(!m("bq|")); + BOOST_CHECK(!m("b*|")); + BOOST_CHECK(!m("")); + } + { + globish_matcher m(utf8("{a,\\\\,b*}"), utf8("*c*")); + BOOST_CHECK(m("a")); + BOOST_CHECK(!m("ab")); + BOOST_CHECK(m("\\")); + BOOST_CHECK(!m("\\\\")); + BOOST_CHECK(m("b")); + BOOST_CHECK(m("bfoobar")); + BOOST_CHECK(!m("bfoobarcfoobar")); + } + { + globish_matcher m(utf8("*"), utf8("")); + BOOST_CHECK(m("foo")); + BOOST_CHECK(m("")); + } + { + globish_matcher m(utf8("{foo}"), utf8("")); + BOOST_CHECK(m("foo")); + BOOST_CHECK(!m("bar")); + } +} + + +void add_globish_tests(test_suite * suite) +{ + I(suite); + suite->add(BOOST_TEST_CASE(&checked_globish_to_regex_test)); + suite->add(BOOST_TEST_CASE(&combine_and_check_globish_test)); + suite->add(BOOST_TEST_CASE(&globish_matcher_test)); +} + +#endif // BUILD_UNIT_TESTS --- globish.hh +++ globish.hh @@ -0,0 +1,47 @@ +#ifndef __GLOBISH_HH__ +#define __GLOBISH_HH__ + +// copyright (C) 2005 nathaniel smith +// all rights reserved. +// licensed to the public under the terms of the GNU GPL (>= 2) +// see the file COPYING for details + +// a sort of glob-like pattern matcher, for use in specifying branch +// collections for netsync. it is important that it not be too expensive to +// match (as opposed to common regex engines, which can be exponential on +// pathological patterns), because we must match branches against untrusted +// patterns when doing netsync. + +// the syntax is: +// most things - match themselves +// * - match 0 or more characters +// ? - match 0 or 1 characters +// \ - match +// {,,...} - match any of the given items +// so like standard globs, except without [] character sets, and with {} +// alternation. +// the one strange thing is there is a special-case -- the empty pattern +// matches nothing, not even the empty string. this hardly ever matters, but +// it's nice to have some way to say "don't exclude anything", for instance. + +#include +#include +#include + +#include "vocab.hh" + +void combine_and_check_globish(std::set const &patterns, utf8 & pattern); + +class globish_matcher +{ +public: + // this may throw an informative_failure if a pattern is invalid + globish_matcher(utf8 const & include_pat, utf8 const & exclude_pat); + // this method may throw a std::runtime_error if the pattern is really + // pathological + bool operator()(std::string const & s); +private: + boost::regex r_inc, r_exc; +}; + +#endif --- hmac.cc +++ hmac.cc @@ -0,0 +1,44 @@ +#include + +#include "cryptopp/hmac.h" +#include "cryptopp/sha.h" + +#include "sanity.hh" +#include "hmac.hh" +#include "vocab.hh" +#include "constants.hh" + +chained_hmac::chained_hmac(netsync_session_key const & session_key) : + key(session_key) +{ + I(hmac_length == CryptoPP::SHA::DIGESTSIZE); + memset(chain_val, 0, sizeof(chain_val)); +} + +void +chained_hmac::set_key(netsync_session_key const & session_key) +{ + key = session_key; +} + +std::string +chained_hmac::process(std::string const & str, size_t pos, size_t n) +{ + I(pos < str.size()); + if (n == std::string::npos) + n = str.size() - pos; + + I(pos + n <= str.size()); + + CryptoPP::HMAC + hmac(reinterpret_cast(key().data()), + constants::netsync_session_key_length_in_bytes); + hmac.Update(reinterpret_cast(chain_val), + sizeof(chain_val)); + hmac.Update(reinterpret_cast(str.data() + pos), + n); + hmac.Final(reinterpret_cast(chain_val)); + + std::string out(chain_val, sizeof(chain_val)); + return out; +} --- hmac.hh +++ hmac.hh @@ -0,0 +1,30 @@ +#ifndef __HMAC_HH__ +#define __HMAC_HH__ + +#include + +#include "cryptopp/hmac.h" +#include "cryptopp/sha.h" + +#include "vocab.hh" + +struct chained_hmac +{ + public: + chained_hmac(netsync_session_key const & session_key); + void set_key(netsync_session_key const & session_key); + std::string process(std::string const & str, size_t pos = 0, + size_t n = std::string::npos); + + static size_t const hmac_length = CryptoPP::SHA::DIGESTSIZE; + + private: + netsync_session_key key; + char chain_val[hmac_length]; +}; + + + + +#endif // __HMAC_HH__ + --- keys.cc +++ keys.cc @@ -166,7 +166,21 @@ der_encoded[i] = '\0'; } +static bool +blocking_rng(lua_hooks & lua) +{ + if (!lua.hook_non_blocking_rng_ok()) + { +#ifndef BLOCKING_RNG_AVAILABLE + throw oops("no blocking RNG available and non-blocking RNG rejected"); +#else + return true; +#endif + }; + return false; +} + void generate_key_pair(lua_hooks & lua, // to hook for phrase rsa_keypair_id const & id, // to prompting user for phrase @@ -176,17 +190,7 @@ { // we will panic here if the user doesn't like urandom and we can't give // them a real entropy-driven random. - bool request_blocking_rng = false; - if (!lua.hook_non_blocking_rng_ok()) - { -#ifndef BLOCKING_RNG_AVAILABLE - throw oops("no blocking RNG available and non-blocking RNG rejected"); -#else - request_blocking_rng = true; -#endif - } - - AutoSeededRandomPool rng(request_blocking_rng); + AutoSeededRandomPool rng(blocking_rng(lua)); SecByteBlock phrase, pubkey, privkey; rsa_pub_key raw_pub_key; arc4 raw_priv_key; @@ -267,16 +271,7 @@ // we will panic here if the user doesn't like urandom and we can't give // them a real entropy-driven random. - bool request_blocking_rng = false; - if (!lua.hook_non_blocking_rng_ok()) - { -#ifndef BLOCKING_RNG_AVAILABLE - throw oops("no blocking RNG available and non-blocking RNG rejected"); -#else - request_blocking_rng = true; -#endif - } - AutoSeededRandomPool rng(request_blocking_rng); + AutoSeededRandomPool rng(blocking_rng(lua)); // we permit the user to relax security here, by caching a decrypted key // (if they permit it) through the life of a program run. this helps when @@ -396,6 +391,75 @@ return vf->GetLastResult(); } +void encrypt_rsa(lua_hooks & lua, + rsa_keypair_id const & id, + base64 & pub_encoded, + std::string const & plaintext, + rsa_oaep_sha_data & ciphertext) +{ + AutoSeededRandomPool rng(blocking_rng(lua)); + + rsa_pub_key pub; + decode_base64(pub_encoded, pub); + SecByteBlock pub_block; + pub_block.Assign(reinterpret_cast(pub().data()), pub().size()); + StringSource keysource(pub_block.data(), pub_block.size(), true); + + shared_ptr encryptor; + encryptor = shared_ptr + (new RSAES_OAEP_SHA_Encryptor(keysource)); + + string ciphertext_string; + StringSource tmp(plaintext, true, + encryptor->CreateEncryptionFilter + (rng, new StringSink(ciphertext_string))); + + ciphertext = rsa_oaep_sha_data(ciphertext_string); +} + +void decrypt_rsa(lua_hooks & lua, + rsa_keypair_id const & id, + base64< arc4 > const & priv, + rsa_oaep_sha_data const & ciphertext, + std::string & plaintext) +{ + AutoSeededRandomPool rng(blocking_rng(lua)); + arc4 decoded_key; + SecByteBlock decrypted_key; + SecByteBlock phrase; + shared_ptr decryptor; + + for (int i = 0; i < 3; i++) + { + bool force = false; + decode_base64(priv, decoded_key); + decrypted_key.Assign(reinterpret_cast(decoded_key().data()), + decoded_key().size()); + get_passphrase(lua, id, phrase, false, force); + + try + { + do_arc4(phrase, decrypted_key); + StringSource keysource(decrypted_key.data(), decrypted_key.size(), true); + decryptor = shared_ptr + (new RSAES_OAEP_SHA_Decryptor(keysource)); + } + catch (...) + { + if (i >= 2) + throw informative_failure("failed to decrypt private RSA key, " + "probably incorrect passphrase"); + // don't use the cache bad one next time + force = true; + continue; + } + } + + StringSource tmp(ciphertext(), true, + decryptor->CreateDecryptionFilter + (rng, new StringSink(plaintext))); +} + void read_pubkey(string const & in, rsa_keypair_id & id, --- keys.hh +++ keys.hh @@ -42,6 +42,18 @@ void require_password(rsa_keypair_id const & id, app_state & app); +void encrypt_rsa(lua_hooks & lua, + rsa_keypair_id const & id, + base64 & pub, + std::string const & plaintext, + rsa_oaep_sha_data & ciphertext); + +void decrypt_rsa(lua_hooks & lua, + rsa_keypair_id const & id, + base64< arc4 > const & priv, + rsa_oaep_sha_data const & ciphertext, + std::string & plaintext); + // netsync stuff void read_pubkey(std::string const & in, --- lua.cc +++ lua.cc @@ -738,14 +738,14 @@ lua_hooks::hook_expand_date(std::string const & sel, std::string & exp) { - exp.clear(); + exp.clear(); bool res= Lua(st) .func("expand_date") .push_str(sel) .call(1,1) .extract_str(exp) .ok(); - return res && exp.size(); + return res && exp.size(); } bool @@ -1038,14 +1038,14 @@ } bool -lua_hooks::hook_get_netsync_read_permitted(std::string const & pattern, +lua_hooks::hook_get_netsync_read_permitted(std::string const & branch, rsa_keypair_id const & identity) { bool permitted = false, exec_ok = false; exec_ok = Lua(st) .func("get_netsync_read_permitted") - .push_str(pattern) + .push_str(branch) .push_str(identity()) .call(2,1) .extract_bool(permitted) @@ -1055,13 +1055,13 @@ } bool -lua_hooks::hook_get_netsync_anonymous_read_permitted(std::string const & pattern) +lua_hooks::hook_get_netsync_anonymous_read_permitted(std::string const & branch) { bool permitted = false, exec_ok = false; exec_ok = Lua(st) .func("get_netsync_anonymous_read_permitted") - .push_str(pattern) + .push_str(branch) .call(1,1) .extract_bool(permitted) .ok(); @@ -1070,16 +1070,14 @@ } bool -lua_hooks::hook_get_netsync_write_permitted(std::string const & pattern, - rsa_keypair_id const & identity) +lua_hooks::hook_get_netsync_write_permitted(rsa_keypair_id const & identity) { bool permitted = false, exec_ok = false; exec_ok = Lua(st) .func("get_netsync_write_permitted") - .push_str(pattern) .push_str(identity()) - .call(2,1) + .call(1,1) .extract_bool(permitted) .ok(); --- lua.hh +++ lua.hh @@ -62,11 +62,10 @@ std::map const & new_results); // network hooks - bool hook_get_netsync_read_permitted(std::string const & pattern, + bool hook_get_netsync_read_permitted(std::string const & branch, rsa_keypair_id const & identity); - bool hook_get_netsync_anonymous_read_permitted(std::string const & pattern); - bool hook_get_netsync_write_permitted(std::string const & pattern, - rsa_keypair_id const & identity); + bool hook_get_netsync_anonymous_read_permitted(std::string const & branch); + bool hook_get_netsync_write_permitted(rsa_keypair_id const & identity); // local repo hooks bool hook_ignore_file(file_path const & p); --- monotone.1 +++ monotone.1 @@ -162,17 +162,17 @@ \fBrefresh_inodeprints\fP Turn on inodeprints mode, and force a cache refresh. .TP -\fBpush\fP \fI[ []]\fP -Push contents of \fI\fP to database on \fI\fP. +\fBpush\fP \fI[ []]\fP +Push contents of \fI\fP to database on \fI\fP. .TP -\fBpull\fP \fI[ []]\fP -Push contents of \fI\fP from database on \fI\fP. +\fBpull\fP \fI[ []]\fP +Push contents of \fI\fP from database on \fI\fP. .TP -\fBsync\fP \fI \fP -Sync contents of \fI\fP with database on \fI\fP. +\fBsync\fP \fI \fP +Sync contents of \fI\fP with database on \fI\fP. .TP -\fBserve\fP \fI[--pid-file=] \fP -Serve contents of \fI\fP at network address \fI\fP. If a +\fBserve\fP \fI[--pid-file=] \fP +Serve contents of \fI\fP at network address \fI\fP. If a --pid-file option is provided on the command line, monotone will store the process id of the server in the specified file. .TP --- monotone.texi +++ monotone.texi @@ -1641,7 +1641,7 @@ return false end -function get_netsync_write_permitted (branch, identity) +function get_netsync_write_permitted (identity) if (identity == "abe@@juicebot.co.jp") then return true end if (identity == "beth@@juicebot.co.jp") then return true end return false @@ -1660,22 +1660,24 @@ @smallexample @group -$ monotone --db=jim.db serve jim-laptop.juicebot.co.jp "jp.co.juicebot.jb7.*" +$ monotone --db=jim.db serve jim-laptop.juicebot.co.jp "jp.co.juicebot.jb7*" @end group @end smallexample This command sets up a single listener loop on the host @code{jim-laptop.juicebot.co.jp}, serving all branches matching address@hidden This will naturally -include the @code{jp.co.juicebot.jb7} branch, and any sub-branches. address@hidden This will naturally include the address@hidden branch, and any sub-branches. The quotes +around @code{"jp.co.juicebot.jb7*"} are there to protect the @code{*} +from expansion by the shell; they have no meaning to monotone. Now Abe decides he wishes to fetch Jim's code. To do this he issues the monotone @code{sync} command: @smallexample @group -monotone --db=abe.db sync jim-laptop.juicebot.co.jp "jp.co.juicebot.jb7.*" -monotone: rebuilding merkle trees for pattern jp.co.juicebot.jb7.* +monotone --db=abe.db sync jim-laptop.juicebot.co.jp "jp.co.juicebot.jb7*" +monotone: rebuilding merkle trees ... monotone: connecting to jim-laptop.juicebot.co.jp monotone: [bytes in: 3200] [bytes out: 673] monotone: successful exchange with jim-laptop.juicebot.co.jp @@ -1820,8 +1822,8 @@ @smallexample @group -$ monotone sync jim-laptop.juicebot.co.jp "jp.co.juicebot.jb7.*" -monotone: rebuilding merkle trees for pattern jp.co.juicebot.jb7.* +$ monotone sync jim-laptop.juicebot.co.jp "jp.co.juicebot.jb7*" +monotone: rebuilding merkle trees ... monotone: including branch jp.co.juicebot.jb7 monotone: [keys: 2] [rcerts: 8] monotone: connecting to jim-laptop.juicebot.co.jp @@ -1835,8 +1837,8 @@ @smallexample @group -$ monotone --db=beth.db sync jim-laptop.juicebot.co.jp "jp.co.juicebot.jb7.*" -monotone: rebuilding merkle trees for pattern jp.co.juicebot.jb7.* +$ monotone --db=beth.db sync jim-laptop.juicebot.co.jp "jp.co.juicebot.jb7*" +monotone: rebuilding merkle trees ... monotone: connecting to jim-laptop.juicebot.co.jp monotone: [bytes in: 3200] [bytes out: 673] monotone: successful exchange with jim-laptop.juicebot.co.jp @@ -1887,8 +1889,8 @@ @smallexample @group -$ monotone sync jim-laptop.juicebot.co.jp "jp.co.juicebot.jb7.*" -monotone: rebuilding merkle trees for pattern jp.co.juicebot.jb7.* +$ monotone sync jim-laptop.juicebot.co.jp "jp.co.juicebot.jb7*" +monotone: rebuilding merkle trees ... monotone: including branch jp.co.juicebot.jb7 monotone: [keys: 3] [rcerts: 12] monotone: connecting to jim-laptop.juicebot.co.jp @@ -2641,9 +2643,9 @@ @item default-server The default server for netsync operations to use. Automatically set by first use of netsync. address@hidden default-pattern -The default regex pattern for netsync operations to use. Automatically -set by first use of netsync. address@hidden default-include-pattern +The default branch glob pattern for netsync operations to use. +Automatically set by first use of netsync. @end table @item known-servers @@ -3047,7 +3049,7 @@ @tab @smallexample @group -$ monotone pull www.foo.com com.foo.wobbler +$ monotone pull www.foo.com com.foo.wobbler* $ monotone checkout --revision=fe37 wobbler @end group @end smallexample @@ -3056,7 +3058,7 @@ The CVS command contacts a network server, retrieves a revision, and stores it in your working copy. There are two cosmetic differences with the monotone command: remote databases are specified by hostnames -and regexes, and revisions are denoted by @sc{sha1} values (or +and globs, and revisions are denoted by @sc{sha1} values (or selectors). There is also one deep difference: pulling revisions into your @@ -3080,7 +3082,7 @@ @smallexample @group $ monotone commit --message="log message" -$ monotone push www.foo.com com.foo.wobbler +$ monotone push www.foo.com com.foo.wobbler* @end group @end smallexample @end multitable @@ -3102,7 +3104,7 @@ @tab @smallexample @group -$ monotone pull www.foo.com com.foo.wobbler +$ monotone pull www.foo.com com.foo.wobbler* $ monotone merge $ monotone update @end group @@ -3653,10 +3655,10 @@ @section Network @ftable @command address@hidden monotone serve @var{address}[:@var{port}] @var{regex} [...] address@hidden monotone pull address@hidden:@var{port}] address@hidden address@hidden monotone push address@hidden:@var{port}] address@hidden address@hidden monotone sync address@hidden:@var{port}] address@hidden address@hidden monotone serve @var{address}[:@var{port}] @var{glob} [...] address@hidden monotone pull address@hidden:@var{port}] address@hidden [...]]] address@hidden monotone push address@hidden:@var{port}] address@hidden [...]]] address@hidden monotone sync address@hidden:@var{port}] address@hidden [...]]] These commands operate the ``netsync'' protocol built into monotone. This is a custom protocol for rapidly synchronizing two @@ -3666,25 +3668,25 @@ The network @var{address} specified in each case should be the same: a host name to listen on, or connect to, optionally followed by a colon -and a port number. The @var{regex} parameter indicates a set of -branches to exchange; every branch which matches @var{regex} exactly +and a port number. The @var{glob} parameters indicate a set of +branches to exchange; every branch which matches a @var{glob} exactly will be indexed and made available for synchronization. -The @command{serve} command can take multiple regexes, and it will -make available all branches matching any of the listed regexes. Different -permissions can be applied to each branch; see the hooks +The @command{serve} command can take multiple globs, and it will make +available all branches matching any of them. Different permissions can +be applied to each branch; see the hooks @code{get_netsync_read_permitted}, @code{get_netsync_write_permitted}, -and @code{get_netsync_anonymous_read_permitted}, all of which take a address@hidden argument (see @ref{Hook Reference}). +and @code{get_netsync_anonymous_read_permitted} (see @ref{Hook +Reference}). -For example, supposing Bob and Alice wish to synchronize their +For example, perhaps Bob and Alice wish to synchronize their @code{net.venge.monotone.win32} and @code{net.venge.monotone.i18n} branches. Supposing Alice's computer has hostname @code{alice.someisp.com}, then Alice might run: @smallexample @group -$ monotone serve alice.someisp.com "net.venge.monotone.*" +$ monotone serve alice.someisp.com "net.venge.monotone*" @end group @end smallexample @@ -3692,19 +3694,19 @@ @smallexample @group -$ monotone sync alice.someisp.com "net.venge.monotone.*" +$ monotone sync alice.someisp.com "net.venge.monotone*" @end group @end smallexample When the operation completes, all branches matching address@hidden will be synchronized between Alice and Bob's address@hidden will be synchronized between Alice and Bob's databases. The @command{pull}, @command{push}, and @command{sync} commands only -require you pass @var{address} and @var{regex} the first time you -use one of them; monotone will memorize this use and in the future -default to the same server and regex. For instance, if Bob wants -to @command{sync} with Alice again, he can simply run: +require you pass @var{address} and @var{glob} the first time you use one +of them; monotone will memorize this use and in the future default to +the same server and glob. For instance, if Bob wants to @command{sync} +with Alice again, he can simply run: @smallexample @group @@ -3713,18 +3715,26 @@ @end smallexample Of course, he can still @command{sync} with other people and other -branches by passing an address or address plus regex on the command +branches by passing an address or address plus globs on the command line; this will not affect his default affinity for Alice. If you ever do want to change your defaults, use @code{monotone unset database -default-server} or @code{monotone unset database default-pattern}; -these will clear your defaults, and cause them to be reset to the next -person you netsync with. +default-server} or @code{monotone unset database +default-include-pattern}; these will clear your defaults, and cause them +to be reset to the next person you netsync with. If a @option{--pid-file} option is specified, the command @command{serve} will create the specified file and record the process identifier of the server in the file. This file can then be read to identify specific monotone server processes. +The syntax for patterns is very simple. @code{*} matches 0 or more +arbitrary characters. @code{?} matches exactly 1 arbitrary character. address@hidden@{foo,bar,address@hidden matches ``foo'', or ``bar'', or ``baz''. These +can be combined arbitrarily. A backslash, @code{\}, can be prefixed to +any character, to match exactly that character --- this might be useful +in case someone, for some odd reason, decides to put a ``*'' into their +branch name. + @end ftable @@ -5492,17 +5502,13 @@ access hook. This hook has no default definition, therefore the default behavior is to deny all anonymous reads. address@hidden get_netsync_write_permitted (@var{branch}, @var{identity}) address@hidden get_netsync_write_permitted (@var{identity}) -Returns @code{true} if a peer authenticated as key @var{identity} -should be allowed to write into your database certs, revisions, -manifests, and files associated with @var{branch}; otherwise @code{false}. -This hook has no default definition, therefore the default behavior is to deny all writes. +Returns @code{true} if a peer authenticated as key @var{identity} should +be allowed to write into your database certs, revisions, manifests, and +files; otherwise @code{false}. This hook has no default definition, +therefore the default behavior is to deny all writes. -Note that if write access is granted for one branch it is effectively -granted for the entire database, as there is currently no way to -restrict that access to only that branch. - Note that the @var{identity} value is a key ID (such as address@hidden@@pobox.com}'') but will correspond to a @emph{unique} key fingerprint (hash) in your database. Monotone will not permit two @@ -6744,20 +6750,20 @@ @item @b{refresh_inodeprints} Turn on inodeprints mode, and force a cache refresh. address@hidden @b{push} @i{ } -Push contents of branches matching @i{} to database on @i{} address@hidden @b{push} @i{ } +Push contents of branches matching @i{} to database on @i{} @comment TROFF INPUT: .SH DESCRIPTION address@hidden @b{pull} @i{ } -Pull contents of branches matching @i{} from database on @i{} address@hidden @b{pull} @i{ } +Pull contents of branches matching @i{} from database on @i{} @comment TROFF INPUT: .SH DESCRIPTION address@hidden @b{sync} @i{ } -Sync contents of branches matching @i{} with database on @i{} address@hidden @b{sync} @i{ } +Sync contents of branches matching @i{} with database on @i{} @comment TROFF INPUT: .SH DESCRIPTION address@hidden @b{serve} @i{ } -Serve contents of branches matching @i{} at network address @i{} address@hidden @b{serve} @i{ } +Serve contents of branches matching @i{} at network address @i{} @comment TROFF INPUT: .SH DESCRIPTION @item @b{automate} @i{(interface_version|heads|ancestors|attributes|parents|descendents|children|graph|erase_ancestors|toposort|ancestry_difference|leaves|inventory|stdio|certs|select)} --- netcmd.cc +++ netcmd.cc @@ -7,8 +7,6 @@ #include #include -#include "cryptopp/gzip.h" - #include "adler32.hh" #include "constants.hh" #include "netcmd.hh" @@ -16,6 +14,7 @@ #include "numeric_vocab.hh" #include "sanity.hh" #include "transforms.hh" +#include "hmac.hh" using namespace std; @@ -49,9 +48,6 @@ cmd_code(bye_cmd) {} -netcmd::netcmd(u8 _version) : version(_version), cmd_code(bye_cmd) -{} - size_t netcmd::encoded_size() { string tmp; @@ -68,61 +64,39 @@ } void -netcmd::write(string & out) const +netcmd::write(string & out, chained_hmac & hmac) const { + size_t oldlen = out.size(); out += static_cast(version); out += static_cast(cmd_code); insert_variable_length_string(payload, out); - adler32 check(reinterpret_cast(payload.data()), payload.size()); - insert_datum_lsb(check.sum(), out); -} -// last should be zero (doesn't mean we're compatible with version 0). -// The nonzero elements are the historical netsync/netcmd versions we can -// interoperate with. For interoperating with newer versions, assume -// compatibility and let the remote host make the call. -static u8 const compatible_versions[] = {4, 0}; - -bool is_compatible(u8 version) -{ - for (u8 const *x = compatible_versions; *x; ++x) - { - if (*x == version) - return true; - } - return false; + string digest = hmac.process(out, oldlen); + I(hmac.hmac_length == constants::netsync_hmac_value_length_in_bytes); + out.append(digest); } - + bool -netcmd::read(string & inbuf) +netcmd::read(string & inbuf, chained_hmac & hmac) { size_t pos = 0; if (inbuf.size() < constants::netcmd_minsz) return false; - u8 ver = extract_datum_lsb(inbuf, pos, "netcmd protocol number"); - int v = version; + u8 extracted_ver = extract_datum_lsb(inbuf, pos, "netcmd protocol number"); + if (extracted_ver != version) + throw bad_decode(F("protocol version mismatch: wanted '%d' got '%d'") + % widen(version) + % widen(extracted_ver)); + version = extracted_ver; u8 cmd_byte = extract_datum_lsb(inbuf, pos, "netcmd code"); switch (cmd_byte) { - // hello may be newer than expected, or one we're compatible with case static_cast(hello_cmd): - if (ver < version && !is_compatible(ver)) - throw bad_decode(F("protocol version mismatch: wanted '%d' got '%d'") - % widen(v) - % widen(ver)); - break; - // these may be older compatible versions case static_cast(anonymous_cmd): case static_cast(auth_cmd): - if (ver != version && (ver > version || !is_compatible(ver))) - throw bad_decode(F("protocol version mismatch: wanted '%d' got '%d'") - % widen(v) - % widen(ver)); - break; - // these must match exactly what's expected case static_cast(error_cmd): case static_cast(bye_cmd): case static_cast(confirm_cmd): @@ -133,16 +107,11 @@ case static_cast(data_cmd): case static_cast(delta_cmd): case static_cast(nonexistant_cmd): - if (ver != version) - throw bad_decode(F("protocol version mismatch: wanted '%d' got '%d'") - % widen(v) - % widen(ver)); + cmd_code = static_cast(cmd_byte); break; default: throw bad_decode(F("unknown netcmd code 0x%x") % widen(cmd_byte)); } - cmd_code = static_cast(cmd_byte); - version = ver; // check to see if we have even enough bytes for a complete uleb128 size_t payload_len = 0; @@ -155,28 +124,35 @@ throw bad_decode(F("oversized payload of '%d' bytes") % payload_len); // there might not be enough data yet in the input buffer - if (inbuf.size() < pos + payload_len + sizeof(u32)) + if (inbuf.size() < pos + payload_len + constants::netsync_hmac_value_length_in_bytes) { - inbuf.reserve(pos + payload_len + sizeof(u32) + constants::bufsz); + inbuf.reserve(pos + payload_len + constants::netsync_hmac_value_length_in_bytes + constants::bufsz); return false; } // out.payload = extract_substring(inbuf, pos, payload_len, "netcmd payload"); // Do this ourselves, so we can swap the strings instead of copying. require_bytes(inbuf, pos, payload_len, "netcmd payload"); - inbuf.erase(0, pos); - payload = inbuf.substr(payload_len); - inbuf.erase(payload_len, inbuf.npos); + + // grab it before the data gets munged + I(hmac.hmac_length == constants::netsync_hmac_value_length_in_bytes); + string digest = hmac.process(inbuf, 0, pos + payload_len); + + payload = inbuf.substr(pos + payload_len); + inbuf.erase(pos + payload_len, inbuf.npos); inbuf.swap(payload); + size_t payload_pos = pos; pos = 0; // they might have given us bogus data - u32 checksum = extract_datum_lsb(inbuf, pos, "netcmd checksum"); + string cmd_digest = extract_substring(inbuf, pos, + constants::netsync_hmac_value_length_in_bytes, + "netcmd HMAC"); inbuf.erase(0, pos); - adler32 check(reinterpret_cast(payload.data()), - payload.size()); - if (checksum != check.sum()) - throw bad_decode(F("bad checksum 0x%x vs. 0x%x") % checksum % check.sum()); + if (cmd_digest != digest) + throw bad_decode(F("bad HMAC %s vs. %s") % encode_hexenc(cmd_digest) + % encode_hexenc(digest)); + payload.erase(0, payload_pos); return true; } @@ -237,112 +213,123 @@ } -void -netcmd::read_anonymous_cmd(protocol_role & role, - std::string & pattern, - id & nonce2) const +void +netcmd::read_anonymous_cmd(protocol_role & role, + utf8 & include_pattern, + utf8 & exclude_pattern, + rsa_oaep_sha_data & hmac_key_encrypted) const { size_t pos = 0; - // syntax is: - u8 role_byte = extract_datum_lsb(payload, pos, "anonymous netcmd, role"); + // syntax is: + u8 role_byte = extract_datum_lsb(payload, pos, "anonymous(hmac) netcmd, role"); if (role_byte != static_cast(source_role) && role_byte != static_cast(sink_role) && role_byte != static_cast(source_and_sink_role)) throw bad_decode(F("unknown role specifier %d") % widen(role_byte)); role = static_cast(role_byte); - extract_variable_length_string(payload, pattern, pos, - "anonymous netcmd, pattern"); - nonce2 = id(extract_substring(payload, pos, - constants::merkle_hash_length_in_bytes, - "anonymous netcmd, nonce2")); - assert_end_of_buffer(payload, pos, "anonymous netcmd payload"); + std::string pattern_string; + extract_variable_length_string(payload, pattern_string, pos, + "anonymous(hmac) netcmd, include_pattern"); + include_pattern = utf8(pattern_string); + extract_variable_length_string(payload, pattern_string, pos, + "anonymous(hmac) netcmd, exclude_pattern"); + exclude_pattern = utf8(pattern_string); + string hmac_key_string; + extract_variable_length_string(payload, hmac_key_string, pos, + "anonymous(hmac) netcmd, hmac_key_encrypted"); + hmac_key_encrypted = rsa_oaep_sha_data(hmac_key_string); + assert_end_of_buffer(payload, pos, "anonymous(hmac) netcmd payload"); } -void -netcmd::write_anonymous_cmd(protocol_role role, - std::string const & pattern, - id const & nonce2) +void +netcmd::write_anonymous_cmd(protocol_role role, + utf8 const & include_pattern, + utf8 const & exclude_pattern, + rsa_oaep_sha_data const & hmac_key_encrypted) { cmd_code = anonymous_cmd; - I(nonce2().size() == constants::merkle_hash_length_in_bytes); payload = static_cast(role); - insert_variable_length_string(pattern, payload); - payload += nonce2(); + insert_variable_length_string(include_pattern(), payload); + insert_variable_length_string(exclude_pattern(), payload); + insert_variable_length_string(hmac_key_encrypted(), payload); } void netcmd::read_auth_cmd(protocol_role & role, - string & pattern, + utf8 & include_pattern, + utf8 & exclude_pattern, id & client, id & nonce1, - id & nonce2, + rsa_oaep_sha_data & hmac_key_encrypted, string & signature) const { size_t pos = 0; - // syntax is: - // - // + // syntax is: + // + // u8 role_byte = extract_datum_lsb(payload, pos, "auth netcmd, role"); if (role_byte != static_cast(source_role) && role_byte != static_cast(sink_role) && role_byte != static_cast(source_and_sink_role)) throw bad_decode(F("unknown role specifier %d") % widen(role_byte)); role = static_cast(role_byte); - extract_variable_length_string(payload, pattern, pos, "auth netcmd, pattern"); + std::string pattern_string; + extract_variable_length_string(payload, pattern_string, pos, + "auth(hmac) netcmd, include_pattern"); + include_pattern = utf8(pattern_string); + extract_variable_length_string(payload, pattern_string, pos, + "auth(hmac) netcmd, exclude_pattern"); + exclude_pattern = utf8(pattern_string); client = id(extract_substring(payload, pos, constants::merkle_hash_length_in_bytes, - "auth netcmd, client identifier")); + "auth(hmac) netcmd, client identifier")); nonce1 = id(extract_substring(payload, pos, constants::merkle_hash_length_in_bytes, - "auth netcmd, nonce1")); - nonce2 = id(extract_substring(payload, pos, - constants::merkle_hash_length_in_bytes, - "auth netcmd, nonce2")); + "auth(hmac) netcmd, nonce1")); + string hmac_key; + extract_variable_length_string(payload, hmac_key, pos, + "auth(hmac) netcmd, hmac_key_encrypted"); + hmac_key_encrypted = rsa_oaep_sha_data(hmac_key); extract_variable_length_string(payload, signature, pos, - "auth netcmd, signature"); - assert_end_of_buffer(payload, pos, "auth netcmd payload"); + "auth(hmac) netcmd, signature"); + assert_end_of_buffer(payload, pos, "auth(hmac) netcmd payload"); } -void -netcmd::write_auth_cmd(protocol_role role, - string const & pattern, +void +netcmd::write_auth_cmd(protocol_role role, + utf8 const & include_pattern, + utf8 const & exclude_pattern, id const & client, - id const & nonce1, - id const & nonce2, + id const & nonce1, + rsa_oaep_sha_data const & hmac_key_encrypted, string const & signature) { cmd_code = auth_cmd; I(client().size() == constants::merkle_hash_length_in_bytes); I(nonce1().size() == constants::merkle_hash_length_in_bytes); - I(nonce2().size() == constants::merkle_hash_length_in_bytes); payload = static_cast(role); - insert_variable_length_string(pattern, payload); + insert_variable_length_string(include_pattern(), payload); + insert_variable_length_string(exclude_pattern(), payload); payload += client(); payload += nonce1(); - payload += nonce2(); + insert_variable_length_string(hmac_key_encrypted(), payload); insert_variable_length_string(signature, payload); } - -void -netcmd::read_confirm_cmd(string & signature) const +void +netcmd::read_confirm_cmd() const { size_t pos = 0; - - // syntax is: - extract_variable_length_string(payload, signature, pos, - "confirm netcmd, signature"); assert_end_of_buffer(payload, pos, "confirm netcmd payload"); } -void -netcmd::write_confirm_cmd(string const & signature) +void +netcmd::write_confirm_cmd() { cmd_code = confirm_cmd; payload.clear(); - insert_variable_length_string(signature, payload); } - + void netcmd::read_refine_cmd(merkle_node & node) const { @@ -449,7 +436,13 @@ extract_variable_length_string(payload, dat, pos, "data netcmd, data payload"); if (compressed_p == 1) - dat = xform(dat); + { + gzip zdat; + data tdat; + zdat.swap(dat); + decode_gzip(zdat, tdat); + tdat.swap(dat); + } assert_end_of_buffer(payload, pos, "data netcmd payload"); } @@ -464,8 +457,10 @@ payload += item(); if (dat.size() > constants::netcmd_minimum_bytes_to_bother_with_gzip) { + gzip zdat; + encode_gzip(dat, zdat); string tmp; - tmp = xform(dat); + zdat.swap(tmp); payload += static_cast(1); // compressed flag insert_variable_length_string(tmp, payload); } @@ -497,8 +492,14 @@ extract_variable_length_string(payload, tmp, pos, "delta netcmd, delta payload"); if (compressed_p == 1) - tmp = xform(tmp); - del = delta(tmp); + { + gzip zdel(tmp); + decode_gzip(zdel, del); + } + else + { + del = tmp; + } assert_end_of_buffer(payload, pos, "delta netcmd payload"); } @@ -514,16 +515,19 @@ payload += base(); payload += ident(); - string tmp = del(); + string tmp; if (tmp.size() > constants::netcmd_minimum_bytes_to_bother_with_gzip) { payload += static_cast(1); // compressed flag - tmp = xform(tmp); + gzip zdel; + encode_gzip(del, zdel); + tmp = zdel(); } else { payload += static_cast(0); // compressed flag + tmp = del(); } I(tmp.size() <= constants::netcmd_payload_limit); insert_variable_length_string(tmp, payload); @@ -558,6 +562,65 @@ #include "transforms.hh" #include +void +test_netcmd_mac() +{ + netcmd out_cmd, in_cmd; + string buf; + netsync_session_key key(constants::netsync_key_initializer); + { + chained_hmac mac(key); + // mutates mac + out_cmd.write(buf, mac); + BOOST_CHECK_THROW(in_cmd.read(buf, mac), bad_decode); + } + + { + chained_hmac mac(key); + out_cmd.write(buf, mac); + } + buf[0] ^= 0xff; + { + chained_hmac mac(key); + BOOST_CHECK_THROW(in_cmd.read(buf, mac), bad_decode); + } + + { + chained_hmac mac(key); + out_cmd.write(buf, mac); + } + buf[buf.size() - 1] ^= 0xff; + { + chained_hmac mac(key); + BOOST_CHECK_THROW(in_cmd.read(buf, mac), bad_decode); + } + + { + chained_hmac mac(key); + out_cmd.write(buf, mac); + } + buf += '\0'; + { + chained_hmac mac(key); + BOOST_CHECK_THROW(in_cmd.read(buf, mac), bad_decode); + } +} + +static void +do_netcmd_roundtrip(netcmd const & out_cmd, netcmd & in_cmd, string & buf) +{ + netsync_session_key key(constants::netsync_key_initializer); + { + chained_hmac mac(key); + out_cmd.write(buf, mac); + } + { + chained_hmac mac(key); + BOOST_CHECK(in_cmd.read(buf, mac)); + } + BOOST_CHECK(in_cmd == out_cmd); +} + void test_netcmd_functions() { @@ -572,10 +635,8 @@ string out_errmsg("your shoelaces are untied"), in_errmsg; string buf; out_cmd.write_error_cmd(out_errmsg); - out_cmd.write(buf); - BOOST_CHECK(in_cmd.read(buf)); + do_netcmd_roundtrip(out_cmd, in_cmd, buf); in_cmd.read_error_cmd(in_errmsg); - BOOST_CHECK(in_cmd == out_cmd); BOOST_CHECK(in_errmsg == out_errmsg); L(F("errmsg_cmd test done, buffer was %d bytes\n") % buf.size()); } @@ -585,9 +646,7 @@ L(F("checking i/o round trip on bye_cmd\n")); netcmd out_cmd, in_cmd; string buf; - out_cmd.write(buf); - BOOST_CHECK(in_cmd.read(buf)); - BOOST_CHECK(in_cmd == out_cmd); + do_netcmd_roundtrip(out_cmd, in_cmd, buf); L(F("bye_cmd test done, buffer was %d bytes\n") % buf.size()); } @@ -600,10 +659,8 @@ rsa_pub_key out_server_key("9387938749238792874"), in_server_key; id out_nonce(raw_sha1("nonce it up")), in_nonce; out_cmd.write_hello_cmd(out_server_keyname, out_server_key, out_nonce); - out_cmd.write(buf); - BOOST_CHECK(in_cmd.read(buf)); + do_netcmd_roundtrip(out_cmd, in_cmd, buf); in_cmd.read_hello_cmd(in_server_keyname, in_server_key, in_nonce); - BOOST_CHECK(in_cmd == out_cmd); BOOST_CHECK(in_server_keyname == out_server_keyname); BOOST_CHECK(in_server_key == out_server_key); BOOST_CHECK(in_nonce == out_nonce); @@ -616,15 +673,18 @@ netcmd out_cmd, in_cmd; protocol_role out_role = source_and_sink_role, in_role; string buf; - id out_nonce2(raw_sha1("nonce start my heart")), in_nonce2; - string out_pattern("radishes galore!"), in_pattern; + // total cheat, since we don't actually verify that rsa_oaep_sha_data + // is sensible anywhere here... + rsa_oaep_sha_data out_key("nonce start my heart"), in_key; + utf8 out_include_pattern("radishes galore!"), in_include_pattern; + utf8 out_exclude_pattern("turnips galore!"), in_exclude_pattern; - out_cmd.write_anonymous_cmd(out_role, out_pattern, out_nonce2); - out_cmd.write(buf); - BOOST_CHECK(in_cmd.read(buf)); - in_cmd.read_anonymous_cmd(in_role, in_pattern, in_nonce2); - BOOST_CHECK(in_cmd == out_cmd); - BOOST_CHECK(in_nonce2 == out_nonce2); + out_cmd.write_anonymous_cmd(out_role, out_include_pattern, out_exclude_pattern, out_key); + do_netcmd_roundtrip(out_cmd, in_cmd, buf); + in_cmd.read_anonymous_cmd(in_role, in_include_pattern, in_exclude_pattern, in_key); + BOOST_CHECK(in_key == out_key); + BOOST_CHECK(in_include_pattern == out_include_pattern); + BOOST_CHECK(in_exclude_pattern == out_exclude_pattern); BOOST_CHECK(in_role == out_role); L(F("anonymous_cmd test done, buffer was %d bytes\n") % buf.size()); } @@ -636,23 +696,26 @@ protocol_role out_role = source_and_sink_role, in_role; string buf; id out_client(raw_sha1("happy client day")), out_nonce1(raw_sha1("nonce me amadeus")), - out_nonce2(raw_sha1("nonce start my heart")), - in_client, in_nonce1, in_nonce2; - string out_signature(raw_sha1("burble") + raw_sha1("gorby")), out_pattern("radishes galore!"), - in_signature, in_pattern; + in_client, in_nonce1; + // total cheat, since we don't actually verify that rsa_oaep_sha_data + // is sensible anywhere here... + rsa_oaep_sha_data out_key("nonce start my heart"), in_key; + string out_signature(raw_sha1("burble") + raw_sha1("gorby")), in_signature; + utf8 out_include_pattern("radishes galore!"), in_include_pattern; + utf8 out_exclude_pattern("turnips galore!"), in_exclude_pattern; - out_cmd.write_auth_cmd(out_role, out_pattern, out_client, out_nonce1, - out_nonce2, out_signature); - out_cmd.write(buf); - BOOST_CHECK(in_cmd.read(buf)); - in_cmd.read_auth_cmd(in_role, in_pattern, in_client, - in_nonce1, in_nonce2, in_signature); - BOOST_CHECK(in_cmd == out_cmd); + out_cmd.write_auth_cmd(out_role, out_include_pattern, out_exclude_pattern + , out_client, out_nonce1, out_key, out_signature); + do_netcmd_roundtrip(out_cmd, in_cmd, buf); + in_cmd.read_auth_cmd(in_role, in_include_pattern, in_exclude_pattern, + in_client, in_nonce1, in_key, in_signature); BOOST_CHECK(in_client == out_client); BOOST_CHECK(in_nonce1 == out_nonce1); - BOOST_CHECK(in_nonce2 == out_nonce2); + BOOST_CHECK(in_key == out_key); BOOST_CHECK(in_signature == out_signature); BOOST_CHECK(in_role == out_role); + BOOST_CHECK(in_include_pattern == out_include_pattern); + BOOST_CHECK(in_exclude_pattern == out_exclude_pattern); L(F("auth_cmd test done, buffer was %d bytes\n") % buf.size()); } @@ -661,14 +724,9 @@ L(F("checking i/o round trip on confirm_cmd\n")); netcmd out_cmd, in_cmd; string buf; - string out_signature(raw_sha1("egg") + raw_sha1("tomago")), in_signature; - - out_cmd.write_confirm_cmd(out_signature); - out_cmd.write(buf); - BOOST_CHECK(in_cmd.read(buf)); - in_cmd.read_confirm_cmd(in_signature); - BOOST_CHECK(in_cmd == out_cmd); - BOOST_CHECK(in_signature == out_signature); + out_cmd.write_confirm_cmd(); + do_netcmd_roundtrip(out_cmd, in_cmd, buf); + in_cmd.read_confirm_cmd(); L(F("confirm_cmd test done, buffer was %d bytes\n") % buf.size()); } @@ -689,10 +747,8 @@ out_node.set_slot_state(15, subtree_state); out_cmd.write_refine_cmd(out_node); - out_cmd.write(buf); - BOOST_CHECK(in_cmd.read(buf)); + do_netcmd_roundtrip(out_cmd, in_cmd, buf); in_cmd.read_refine_cmd(in_node); - BOOST_CHECK(in_cmd == out_cmd); BOOST_CHECK(in_node == out_node); L(F("refine_cmd test done, buffer was %d bytes\n") % buf.size()); } @@ -706,8 +762,7 @@ string buf; out_cmd.write_done_cmd(out_level, out_type); - out_cmd.write(buf); - BOOST_CHECK(in_cmd.read(buf)); + do_netcmd_roundtrip(out_cmd, in_cmd, buf); in_cmd.read_done_cmd(in_level, in_type); BOOST_CHECK(in_level == out_level); BOOST_CHECK(in_type == out_type); @@ -723,8 +778,7 @@ string buf; out_cmd.write_send_data_cmd(out_type, out_id); - out_cmd.write(buf); - BOOST_CHECK(in_cmd.read(buf)); + do_netcmd_roundtrip(out_cmd, in_cmd, buf); in_cmd.read_send_data_cmd(in_type, in_id); BOOST_CHECK(in_type == out_type); BOOST_CHECK(in_id == out_id); @@ -741,8 +795,7 @@ string buf; out_cmd.write_send_delta_cmd(out_type, out_head, out_base); - out_cmd.write(buf); - BOOST_CHECK(in_cmd.read(buf)); + do_netcmd_roundtrip(out_cmd, in_cmd, buf); in_cmd.read_send_delta_cmd(in_type, in_head, in_base); BOOST_CHECK(in_type == out_type); BOOST_CHECK(in_head == out_head); @@ -759,8 +812,7 @@ string out_dat("thank you for flying northwest"), in_dat; string buf; out_cmd.write_data_cmd(out_type, out_id, out_dat); - out_cmd.write(buf); - BOOST_CHECK(in_cmd.read(buf)); + do_netcmd_roundtrip(out_cmd, in_cmd, buf); in_cmd.read_data_cmd(in_type, in_id, in_dat); BOOST_CHECK(in_id == out_id); BOOST_CHECK(in_dat == out_dat); @@ -778,8 +830,7 @@ string buf; out_cmd.write_delta_cmd(out_type, out_head, out_base, out_delta); - out_cmd.write(buf); - BOOST_CHECK(in_cmd.read(buf)); + do_netcmd_roundtrip(out_cmd, in_cmd, buf); in_cmd.read_delta_cmd(in_type, in_head, in_base, in_delta); BOOST_CHECK(in_type == out_type); BOOST_CHECK(in_head == out_head); @@ -797,8 +848,7 @@ string buf; out_cmd.write_nonexistant_cmd(out_type, out_id); - out_cmd.write(buf); - BOOST_CHECK(in_cmd.read(buf)); + do_netcmd_roundtrip(out_cmd, in_cmd, buf); in_cmd.read_nonexistant_cmd(in_type, in_id); BOOST_CHECK(in_type == out_type); BOOST_CHECK(in_id == out_id); @@ -817,6 +867,7 @@ add_netcmd_tests(test_suite * suite) { suite->add(BOOST_TEST_CASE(&test_netcmd_functions)); + suite->add(BOOST_TEST_CASE(&test_netcmd_mac)); } #endif // BUILD_UNIT_TESTS --- netcmd.hh +++ netcmd.hh @@ -12,6 +12,7 @@ #include "merkle_tree.hh" #include "numeric_vocab.hh" #include "vocab.hh" +#include "hmac.hh" typedef enum { @@ -62,8 +63,10 @@ // basic cmd i/o (including checksums) - void write(std::string & out) const; - bool read(std::string & inbuf); + void write(std::string & out, + chained_hmac & hmac) const; + bool read(std::string & inbuf, + chained_hmac & hmac); // i/o functions for each type of command payload void read_error_cmd(std::string & errmsg) const; @@ -79,28 +82,32 @@ rsa_pub_key const & server_key, id const & nonce); - void read_anonymous_cmd(protocol_role & role, - std::string & pattern, - id & nonce2) const; + void read_anonymous_cmd(protocol_role & role, + utf8 & include_pattern, + utf8 & exclude_pattern, + rsa_oaep_sha_data & hmac_key_encrypted) const; void write_anonymous_cmd(protocol_role role, - std::string const & pattern, - id const & nonce2); + utf8 const & include_pattern, + utf8 const & exclude_pattern, + rsa_oaep_sha_data const & hmac_key_encrypted); void read_auth_cmd(protocol_role & role, - std::string & pattern, + utf8 & include_pattern, + utf8 & exclude_pattern, id & client, id & nonce1, - id & nonce2, + rsa_oaep_sha_data & hmac_key_encrypted, std::string & signature) const; void write_auth_cmd(protocol_role role, - std::string const & pattern, + utf8 const & include_pattern, + utf8 const & exclude_pattern, id const & client, id const & nonce1, - id const & nonce2, + rsa_oaep_sha_data const & hmac_key_encrypted, std::string const & signature); - void read_confirm_cmd(std::string & signature) const; - void write_confirm_cmd(std::string const & signature); + void read_confirm_cmd() const; + void write_confirm_cmd(); void read_refine_cmd(merkle_node & node) const; void write_refine_cmd(merkle_node const & node); --- netsync.cc +++ netsync.cc @@ -7,6 +7,7 @@ #include #include #include +#include #include @@ -32,6 +33,8 @@ #include "xdelta.hh" #include "epoch.hh" #include "platform.hh" +#include "hmac.hh" +#include "globish.hh" #include "cryptopp/osrng.h" @@ -112,18 +115,33 @@ // protocol // -------- // +// The protocol is a simple binary command-packet system over TCP; +// each packet consists of a single byte which identifies the protocol +// version, a byte which identifies the command name inside that +// version, a size_t sent as a uleb128 indicating the length of the +// packet, that many bytes of payload, and finally 20 bytes of SHA-1 +// HMAC calculated over the payload. The key for the SHA-1 HMAC is 20 +// bytes of 0 during authentication, and a 20-byte random key chosen +// by the client after authentication (discussed below). +// +//---- Pre-v5 packet format ---- +// // the protocol is a simple binary command-packet system over tcp; each // packet consists of a byte which identifies the protocol version, a byte // which identifies the command name inside that version, a size_t sent as // a uleb128 indicating the length of the packet, and then that many bytes // of payload, and finally 4 bytes of adler32 checksum (in LSB order) over -// the payload. decoding involves simply buffering until a sufficient -// number of bytes are received, then advancing the buffer pointer. any -// time an adler32 check fails, the protocol is assumed to have lost -// synchronization, and the connection is dropped. the parties are free to -// drop the tcp stream at any point, if too much data is received or too -// much idle time passes; no commitments or transactions are made. +// the payload. // +// ---- end pre-v5 packet format ---- +// +// decoding involves simply buffering until a sufficient number of bytes are +// received, then advancing the buffer pointer. any time an integrity check +// (the HMAC) fails, the protocol is assumed to have lost synchronization, and +// the connection is dropped. the parties are free to drop the tcp stream at +// any point, if too much data is received or too much idle time passes; no +// commitments or transactions are made. +// // one special command, "bye", is used to shut down a connection // gracefully. once each side has received all the data they want, they // can send a "bye" command to the other side. as soon as either side has @@ -135,17 +153,27 @@ // "hello " command, which identifies the server's RSA key and // issues a nonce which must be used for a subsequent authentication. // -// the client can then respond with an "auth (source|sink|both) -// " command which identifies its -// RSA key, notes the role it wishes to play in the synchronization, -// identifies the pattern it wishes to sync with, signs the previous -// nonce with its own key, and issues a nonce of its own for mutual -// authentication. +// The client then responds with either: // -// the server can then respond with a "confirm " command, which is -// the signature of the second nonce sent by the client. this -// transitions the peers into an authenticated state and begins refinement. +// An "auth (source|sink|both) +// " command, which identifies its RSA key, notes the +// role it wishes to play in the synchronization, identifies the pattern it +// wishes to sync with, signs the previous nonce with its own key, and informs +// the server of the HMAC key it wishes to use for this session (encrypted +// with the server's public key); or // +// An "anonymous (source|sink|both) +// " command, which identifies the role it wishes to play in the +// synchronization, the pattern it ishes to sync with, and the HMAC key it +// wishes to use for this session (also encrypted with the server's public +// key). +// +// The server then replies with a "confirm" command, which contains no +// other data but will only have the correct HMAC integrity code if +// the server received and properly decrypted the HMAC key offered by +// the client. This transitions the peers into an authenticated state +// and begins refinement. +// // refinement begins with the client sending its root public key and // manifest certificate merkle nodes to the server. the server then // compares the root to each slot in *its* root node, and for each slot @@ -211,7 +239,9 @@ { protocol_role role; protocol_voice const voice; - vector patterns; + utf8 const & our_include_pattern; + utf8 const & our_exclude_pattern; + globish_matcher our_matcher; app_state & app; string peer_id; @@ -219,17 +249,22 @@ Netxx::Stream str; string inbuf; - string outbuf; + // deque of pair + deque< pair > outbuf; + // the total data stored in outbuf - this is + // used as a valve to stop too much data + // backing up + size_t outbuf_size; netcmd cmd; - u8 protocol_version; bool armed; bool arm(); - utf8 pattern; - boost::regex pattern_re; id remote_peer_key_hash; rsa_keypair_id remote_peer_key_name; + netsync_session_key session_key; + chained_hmac read_hmac; + chained_hmac write_hmac; bool authenticated; time_t last_io_time; @@ -263,7 +298,8 @@ session(protocol_role role, protocol_voice voice, - vector const & patterns, + utf8 const & our_include_pattern, + utf8 const & our_exclude_pattern, app_state & app, string const & peer, Netxx::socket_type sock, @@ -278,6 +314,8 @@ id mk_nonce(); void mark_recent_io(); + void set_session_key(string const & key); + void setup_client_tickers(); bool done_all_refinements(); @@ -321,15 +359,19 @@ void queue_hello_cmd(id const & server, id const & nonce); void queue_anonymous_cmd(protocol_role role, - string const & pattern, - id const & nonce2); + utf8 const & include_pattern, + utf8 const & exclude_pattern, + id const & nonce2, + base64 server_key_encoded); void queue_auth_cmd(protocol_role role, - string const & pattern, + utf8 const & include_pattern, + utf8 const & exclude_pattern, id const & client, id const & nonce1, id const & nonce2, - string const & signature); - void queue_confirm_cmd(string const & signature); + string const & signature, + base64 server_key_encoded); + void queue_confirm_cmd(); void queue_refine_cmd(merkle_node const & node); void queue_send_data_cmd(netcmd_item_type type, id const & item); @@ -353,15 +395,17 @@ rsa_pub_key const & server_key, id const & nonce); bool process_anonymous_cmd(protocol_role role, - string const & pattern, - id const & nonce2); + utf8 const & their_include_pattern, + utf8 const & their_exclude_pattern); bool process_auth_cmd(protocol_role role, - string const & pattern, + utf8 const & their_include_pattern, + utf8 const & their_exclude_pattern, id const & client, id const & nonce1, - id const & nonce2, string const & signature); + void respond_to_auth_cmd(rsa_oaep_sha_data hmac_key_encrypted); bool process_confirm_cmd(string const & signature); + void respond_to_confirm_cmd(); bool process_refine_cmd(merkle_node const & node); bool process_send_data_cmd(netcmd_item_type type, id const & item); @@ -420,26 +464,29 @@ session::session(protocol_role role, protocol_voice voice, - vector const & patterns, + utf8 const & our_include_pattern, + utf8 const & our_exclude_pattern, app_state & app, string const & peer, Netxx::socket_type sock, Netxx::Timeout const & to) : role(role), voice(voice), - patterns(patterns), + our_include_pattern(our_include_pattern), + our_exclude_pattern(our_exclude_pattern), + our_matcher(our_include_pattern, our_exclude_pattern), app(app), peer_id(peer), fd(sock), str(sock, to), inbuf(""), - outbuf(""), - protocol_version(constants::netcmd_current_protocol_version), + outbuf_size(0), armed(false), - pattern(""), - pattern_re(".*"), remote_peer_key_hash(""), remote_peer_key_name(""), + session_key(constants::netsync_key_initializer), + read_hmac(constants::netsync_key_initializer), + write_hmac(constants::netsync_key_initializer), authenticated(false), last_io_time(::time(NULL)), byte_in_ticker(NULL), @@ -456,14 +503,6 @@ dbw(app, true), encountered_error(false) { - if (voice == client_voice) - { - N(patterns.size() == 1, - F("client can only sync one pattern at a time")); - this->pattern = idx(patterns, 0); - this->pattern_re = boost::regex(this->pattern()); - } - dbw.set_on_revision_written(boost::bind(&session::rev_written_callback, this, _1)); dbw.set_on_cert_written(boost::bind(&session::cert_written_callback, @@ -582,6 +621,14 @@ } void +session::set_session_key(string const & key) +{ + session_key = netsync_session_key(key); + read_hmac.set_key(session_key); + write_hmac.set_key(session_key); +} + +void session::setup_client_tickers() { byte_in_ticker.reset(new ticker("bytes in", ">", 1024, true)); @@ -730,7 +777,12 @@ session::write_netcmd_and_try_flush(netcmd const & cmd) { if (!encountered_error) - cmd.write(outbuf); + { + string buf; + cmd.write(buf, write_hmac); + outbuf.push_back(make_pair(buf, 0)); + outbuf_size += buf.size(); + } else L(F("dropping outgoing netcmd (because we're in error unwind mode)\n")); // FIXME: this helps keep the protocol pipeline full but it seems to @@ -1108,8 +1160,6 @@ // Write permissions checking: // remove heads w/o proper certs, add their children to heads // 1) remove unwanted branch certs from consideration - // - server: check write permission hook - // - client: check against sync pattern // 2) remove heads w/o a branch tag, process new exposed heads // 3) repeat 2 until no change @@ -1133,13 +1183,7 @@ ; else { - bool ok; - if (voice == server_voice) - ok = app.lua.hook_get_netsync_write_permitted(name(), - remote_peer_key_name); - else - ok = boost::regex_match(name(), pattern_re); - if (ok) + if (our_matcher(name())) { ok_branches.insert(name()); keeping.push_back(*j); @@ -1288,14 +1332,23 @@ session::write_some() { I(!outbuf.empty()); - Netxx::signed_size_type count = str.write(outbuf.data(), - std::min(outbuf.size(), + size_t writelen = outbuf.front().first.size() - outbuf.front().second; + Netxx::signed_size_type count = str.write(outbuf.front().first.data() + outbuf.front().second, + std::min(writelen, constants::bufsz)); if (count > 0) { - outbuf.erase(0, count); - L(F("wrote %d bytes to fd %d (peer %s), %d remain in output buffer\n") - % count % fd % peer_id % outbuf.size()); + if ((size_t)count == writelen) + { + outbuf_size -= outbuf.front().first.size(); + outbuf.pop_front(); + } + else + { + outbuf.front().second += count; + } + L(F("wrote %d bytes to fd %d (peer %s)\n") + % count % fd % peer_id); mark_recent_io(); if (byte_out_ticker.get() != NULL) (*byte_out_ticker) += count; @@ -1317,7 +1370,7 @@ session::queue_bye_cmd() { L(F("queueing 'bye' command\n")); - netcmd cmd(protocol_version); + netcmd cmd; cmd.write_bye_cmd(); write_netcmd_and_try_flush(cmd); this->sent_goodbye = true; @@ -1327,7 +1380,7 @@ session::queue_error_cmd(string const & errmsg) { L(F("queueing 'error' command\n")); - netcmd cmd(protocol_version); + netcmd cmd; cmd.write_error_cmd(errmsg); write_netcmd_and_try_flush(cmd); this->sent_goodbye = true; @@ -1340,7 +1393,7 @@ string typestr; netcmd_item_type_to_string(type, typestr); L(F("queueing 'done' command for %s level %s\n") % typestr % level); - netcmd cmd(protocol_version); + netcmd cmd; cmd.write_done_cmd(level, type); write_netcmd_and_try_flush(cmd); } @@ -1349,7 +1402,7 @@ session::queue_hello_cmd(id const & server, id const & nonce) { - netcmd cmd(protocol_version); + netcmd cmd; hexenc server_encoded; encode_hexenc(server, server_encoded); @@ -1365,32 +1418,46 @@ void session::queue_anonymous_cmd(protocol_role role, - string const & pattern, - id const & nonce2) + utf8 const & include_pattern, + utf8 const & exclude_pattern, + id const & nonce2, + base64 server_key_encoded) { - netcmd cmd(protocol_version); - cmd.write_anonymous_cmd(role, pattern, nonce2); + netcmd cmd; + rsa_oaep_sha_data hmac_key_encrypted; + encrypt_rsa(app.lua, remote_peer_key_name, server_key_encoded, + nonce2(), hmac_key_encrypted); + cmd.write_anonymous_cmd(role, include_pattern, exclude_pattern, + hmac_key_encrypted); write_netcmd_and_try_flush(cmd); + set_session_key(nonce2()); } void session::queue_auth_cmd(protocol_role role, - string const & pattern, + utf8 const & include_pattern, + utf8 const & exclude_pattern, id const & client, id const & nonce1, id const & nonce2, - string const & signature) + string const & signature, + base64 server_key_encoded) { - netcmd cmd(protocol_version); - cmd.write_auth_cmd(role, pattern, client, nonce1, nonce2, signature); + netcmd cmd; + rsa_oaep_sha_data hmac_key_encrypted; + encrypt_rsa(app.lua, remote_peer_key_name, server_key_encoded, + nonce2(), hmac_key_encrypted); + cmd.write_auth_cmd(role, include_pattern, exclude_pattern, client, + nonce1, hmac_key_encrypted, signature); write_netcmd_and_try_flush(cmd); + set_session_key(nonce2()); } -void -session::queue_confirm_cmd(string const & signature) +void +session::queue_confirm_cmd() { - netcmd cmd(protocol_version); - cmd.write_confirm_cmd(signature); + netcmd cmd; + cmd.write_confirm_cmd(); write_netcmd_and_try_flush(cmd); } @@ -1403,7 +1470,7 @@ netcmd_item_type_to_string(node.type, typestr); L(F("queueing request for refinement of %s node '%s', level %d\n") % typestr % hpref % static_cast(node.level)); - netcmd cmd(protocol_version); + netcmd cmd; cmd.write_refine_cmd(node); write_netcmd_and_try_flush(cmd); } @@ -1433,7 +1500,7 @@ L(F("queueing request for data of %s item '%s'\n") % typestr % hid); - netcmd cmd(protocol_version); + netcmd cmd; cmd.write_send_data_cmd(type, item); write_netcmd_and_try_flush(cmd); note_item_requested(type, item); @@ -1469,7 +1536,7 @@ L(F("queueing request for contents of %s delta '%s' -> '%s'\n") % typestr % base_hid % ident_hid); - netcmd cmd(protocol_version); + netcmd cmd; cmd.write_send_delta_cmd(type, base, ident); write_netcmd_and_try_flush(cmd); note_item_requested(type, ident); @@ -1494,7 +1561,7 @@ L(F("queueing %d bytes of data for %s item '%s'\n") % dat.size() % typestr % hid); - netcmd cmd(protocol_version); + netcmd cmd; cmd.write_data_cmd(type, item, dat); write_netcmd_and_try_flush(cmd); note_item_sent(type, item); @@ -1524,7 +1591,7 @@ L(F("queueing %s delta '%s' -> '%s'\n") % typestr % base_hid % ident_hid); - netcmd cmd(protocol_version); + netcmd cmd; cmd.write_delta_cmd(type, base, ident, del); write_netcmd_and_try_flush(cmd); note_item_sent(type, ident); @@ -1547,7 +1614,7 @@ L(F("queueing note of nonexistance of %s item '%s'\n") % typestr % hid); - netcmd cmd(protocol_version); + netcmd cmd; cmd.write_nonexistant_cmd(type, item); write_netcmd_and_try_flush(cmd); } @@ -1633,22 +1700,6 @@ W(F("No branches found.")); } -void -convert_pattern(utf8 & pat, utf8 & conv) -{ - string x = pat(); - string pattern = ""; - string e = ".|*?+()[]{}^$\\"; - for (string::const_iterator i = x.begin(); i != x.end(); i++) - { - if (e.find(*i) != e.npos) - pattern += '\\'; - pattern += *i; - } - conv = pattern + ".*"; -} - - static const var_domain known_servers_domain = var_domain("known-servers"); bool @@ -1712,21 +1763,13 @@ this->remote_peer_key_hash = their_key_hash_decoded; } - utf8 pat(pattern); - if (protocol_version < 5) - { - W(F("Talking to an old server. " - "Using %s as a collection, not a regex.") % pattern); - convert_pattern(pattern, pat); - this->pattern_re = boost::regex(pat()); - } vector branchnames; set ok_branches; get_branches(app, branchnames); for (vector::const_iterator i = branchnames.begin(); i != branchnames.end(); i++) { - if (boost::regex_match(*i, pattern_re)) + if (our_matcher(*i)) ok_branches.insert(utf8(*i)); } rebuild_merkle_trees(app, ok_branches); @@ -1752,41 +1795,23 @@ decode_base64(sig, sig_raw); // make a new nonce of our own and send off the 'auth' - queue_auth_cmd(this->role, this->pattern(), our_key_hash_raw, - nonce, mk_nonce(), sig_raw()); + queue_auth_cmd(this->role, our_include_pattern, our_exclude_pattern, + our_key_hash_raw, nonce, mk_nonce(), sig_raw(), + their_key_encoded); } else { - queue_anonymous_cmd(this->role, this->pattern(), mk_nonce()); + queue_anonymous_cmd(this->role, our_include_pattern, + our_exclude_pattern, mk_nonce(), their_key_encoded); } return true; } -bool -matches_one(string s, vector r) -{ - for (vector::const_iterator i = r.begin(); i != r.end(); i++) - { - if (boost::regex_match(s, *i)) - return true; - } - return false; -} - bool session::process_anonymous_cmd(protocol_role role, - string const & pattern, - id const & nonce2) + utf8 const & their_include_pattern, + utf8 const & their_exclude_pattern) { - hexenc hnonce2; - encode_hexenc(nonce2, hnonce2); - - L(F("received 'anonymous' netcmd from client for pattern '%s' " - "in %s mode with nonce2 '%s'\n") - % pattern % (role == source_and_sink_role ? "source and sink" : - (role == source_role ? "source " : "sink")) - % hnonce2); - // // internally netsync thinks in terms of sources and sinks. users like // thinking of repositories as "readonly", "readwrite", or "writeonly". @@ -1819,83 +1844,51 @@ vector branchnames; set ok_branches; get_branches(app, branchnames); - vector allowed; - boost::regex reg(pattern); - for (vector::const_iterator i = patterns.begin(); - i != patterns.end(); ++i) - { - allowed.push_back(boost::regex((*i)())); - } + globish_matcher their_matcher(their_include_pattern, their_exclude_pattern); for (vector::const_iterator i = branchnames.begin(); i != branchnames.end(); i++) { - if (boost::regex_match(*i, reg) - && (allowed.size() == 0 || matches_one(*i, allowed))) - { - if (app.lua.hook_get_netsync_anonymous_read_permitted(*i)) - ok_branches.insert(utf8(*i)); - } + if (our_matcher(*i) && their_matcher(*i) + && app.lua.hook_get_netsync_anonymous_read_permitted(*i)) + ok_branches.insert(utf8(*i)); } - if (!ok_branches.size()) + if (ok_branches.empty()) { - W(F("denied anonymous read permission for '%s'\n") % pattern); + W(F("denied anonymous read permission for '%s' excluding '%s'\n") + % their_include_pattern % their_exclude_pattern); this->saved_nonce = id(""); return false; } - P(F("allowed anonymous read permission for '%s'\n") % pattern); + P(F("allowed anonymous read permission for '%s' excluding '%s'\n") + % their_include_pattern % their_exclude_pattern); rebuild_merkle_trees(app, ok_branches); - // get our private key and sign back - L(F("anonymous read permitted, signing back nonce\n")); - base64 sig; - rsa_sha1_signature sig_raw; - base64< arc4 > our_priv; - load_priv_key(app, app.signing_key, our_priv); - make_signature(app.lua, app.signing_key, our_priv, nonce2(), sig); - decode_base64(sig, sig_raw); - queue_confirm_cmd(sig_raw()); - this->pattern = pattern; this->remote_peer_key_name = rsa_keypair_id(""); this->authenticated = true; this->role = source_role; return true; } -bool -session::process_auth_cmd(protocol_role role, - string const & pattern, - id const & client, - id const & nonce1, - id const & nonce2, +bool +session::process_auth_cmd(protocol_role their_role, + utf8 const & their_include_pattern, + utf8 const & their_exclude_pattern, + id const & client, + id const & nonce1, string const & signature) { I(this->remote_peer_key_hash().size() == 0); I(this->saved_nonce().size() == constants::merkle_hash_length_in_bytes); - hexenc hnonce1, hnonce2; - encode_hexenc(nonce1, hnonce1); - encode_hexenc(nonce2, hnonce2); hexenc their_key_hash; encode_hexenc(client, their_key_hash); set ok_branches; vector branchnames; get_branches(app, branchnames); - vector allowed; - for (vector::const_iterator i = patterns.begin(); - i != patterns.end(); ++i) - { - allowed.push_back(boost::regex((*i)())); - } - boost::regex reg(pattern); + globish_matcher their_matcher(their_include_pattern, their_exclude_pattern); - L(F("received 'auth' netcmd from client '%s' for pattern '%s' " - "in %s mode with nonce1 '%s' and nonce2 '%s'\n") - % their_key_hash % pattern % (role == source_and_sink_role ? "source and sink" : - (role == source_role ? "source " : "sink")) - % hnonce1 % hnonce2); - // check that they replied with the nonce we asked for if (!(nonce1 == this->saved_nonce)) { @@ -1913,7 +1906,7 @@ // if the user asks to run a "read only" service, this means they are // willing to be a source but not a sink. // - // nb: the "role" here is the role the *client* wants to play + // nb: the "their_role" here is the role the *client* wants to play // so we need to check that the opposite role is allowed for us, // in our this->role field. // @@ -1932,12 +1925,12 @@ // client as sink, server as source (reading) - if (role == sink_role || role == source_and_sink_role) + if (their_role == sink_role || their_role == source_and_sink_role) { if (this->role != source_role && this->role != source_and_sink_role) { - W(F("denied '%s' read permission for '%s' while running as sink\n") - % their_id % pattern); + W(F("denied '%s' read permission for '%s' excluding '%s' while running as pure sink\n") + % their_id % their_include_pattern % their_exclude_pattern); this->saved_nonce = id(""); return false; } @@ -1945,48 +1938,46 @@ for (vector::const_iterator i = branchnames.begin(); i != branchnames.end(); i++) { - if (boost::regex_match(*i, reg) - && (allowed.size() == 0 || matches_one(*i, allowed))) - { - if (app.lua.hook_get_netsync_read_permitted(*i, their_id)) - ok_branches.insert(utf8(*i)); - } + if (our_matcher(*i) && their_matcher(*i) + && app.lua.hook_get_netsync_read_permitted(*i, their_id)) + ok_branches.insert(utf8(*i)); } //if we're source_and_sink_role, continue even with no branches readable //ex: serve --db=empty.db - if (!ok_branches.size() && role == sink_role) + if (ok_branches.empty() && their_role == sink_role) { - W(F("denied '%s' read permission for '%s'\n") % their_id % pattern); + W(F("denied '%s' read permission for '%s' excluding '%s'\n") + % their_id % their_include_pattern % their_exclude_pattern); this->saved_nonce = id(""); return false; } - P(F("allowed '%s' read permission for '%s'\n") % their_id % pattern); + P(F("allowed '%s' read permission for '%s' excluding '%s'\n") + % their_id % their_include_pattern % their_exclude_pattern); } // client as source, server as sink (writing) - if (role == source_role || role == source_and_sink_role) + if (their_role == source_role || their_role == source_and_sink_role) { if (this->role != sink_role && this->role != source_and_sink_role) { - W(F("denied '%s' write permission for '%s' while running as source\n") - % their_id % pattern); + W(F("denied '%s' write permission for '%s' excluding '%s' while running as pure source\n") + % their_id % their_include_pattern % their_exclude_pattern); this->saved_nonce = id(""); return false; } - // Write permissions are now checked from analyze_ancestry_graph. - if (role == source_role) + if (!app.lua.hook_get_netsync_write_permitted(their_id)) { - for (vector::const_iterator i = branchnames.begin(); - i != branchnames.end(); i++) - { - ok_branches.insert(utf8(*i)); - } + W(F("denied '%s' write permission for '%s' excluding '%s' while running as pure source\n") + % their_id % their_include_pattern % their_exclude_pattern); + this->saved_nonce = id(""); + return false; } - P(F("allowed '%s' write permission for '%s'\n") % their_id % pattern); + P(F("allowed '%s' write permission for '%s' excluding '%s'\n") + % their_id % their_include_pattern % their_exclude_pattern); } rebuild_merkle_trees(app, ok_branches); @@ -2001,19 +1992,10 @@ { // get our private key and sign back L(F("client signature OK, accepting authentication\n")); - base64 sig; - rsa_sha1_signature sig_raw; - base64< arc4 > our_priv; - load_priv_key(app, app.signing_key, our_priv); - make_signature(app.lua, app.signing_key, our_priv, nonce2(), sig); - decode_base64(sig, sig_raw); - queue_confirm_cmd(sig_raw()); - this->pattern = pattern; - this->pattern_re = boost::regex(this->pattern()); this->authenticated = true; this->remote_peer_key_name = their_id; // assume the (possibly degraded) opposite role - switch (role) + switch (their_role) { case source_role: I(this->role != source_role); @@ -2036,6 +2018,18 @@ return false; } +void +session::respond_to_auth_cmd(rsa_oaep_sha_data hmac_key_encrypted) +{ + L(F("Writing HMAC confirm command")); + base64< arc4 > our_priv; + load_priv_key(app, app.signing_key, our_priv); + string hmac_key; + decrypt_rsa(app.lua, app.signing_key, our_priv, hmac_key_encrypted, hmac_key); + set_session_key(hmac_key); + queue_confirm_cmd(); +} + bool session::process_confirm_cmd(string const & signature) { @@ -2046,9 +2040,10 @@ encode_hexenc(id(remote_peer_key_hash), their_key_hash); // nb. this->role is our role, the server is in the opposite role - L(F("received 'confirm' netcmd from server '%s' for pattern '%s' in %s mode\n") - % their_key_hash % this->pattern % (this->role == source_and_sink_role ? "source and sink" : - (this->role == source_role ? "sink" : "source"))); + L(F("received 'confirm' netcmd from server '%s' for pattern '%s' exclude '%s' in %s mode\n") + % their_key_hash % our_include_pattern % our_exclude_pattern + % (this->role == source_and_sink_role ? "source and sink" : + (this->role == source_role ? "sink" : "source"))); // check their signature if (app.db.public_key_exists(their_key_hash)) @@ -2062,20 +2057,6 @@ if (check_signature(app.lua, their_id, their_key, this->saved_nonce(), sig)) { L(F("server signature OK, accepting authentication\n")); - this->authenticated = true; - - merkle_ptr root; - load_merkle_node(epoch_item, 0, get_root_prefix().val, root); - queue_refine_cmd(*root); - queue_done_cmd(0, epoch_item); - - load_merkle_node(key_item, 0, get_root_prefix().val, root); - queue_refine_cmd(*root); - queue_done_cmd(0, key_item); - - load_merkle_node(cert_item, 0, get_root_prefix().val, root); - queue_refine_cmd(*root); - queue_done_cmd(0, cert_item); return true; } else @@ -2090,6 +2071,23 @@ return false; } +void +session::respond_to_confirm_cmd() +{ + merkle_ptr root; + load_merkle_node(epoch_item, 0, get_root_prefix().val, root); + queue_refine_cmd(*root); + queue_done_cmd(0, epoch_item); + + load_merkle_node(key_item, 0, get_root_prefix().val, root); + queue_refine_cmd(*root); + queue_done_cmd(0, key_item); + + load_merkle_node(cert_item, 0, get_root_prefix().val, root); + queue_refine_cmd(*root); + queue_done_cmd(0, cert_item); +} + static bool data_exists(netcmd_item_type type, id const & item, @@ -2982,8 +2980,6 @@ rsa_pub_key server_key; id nonce; cmd.read_hello_cmd(server_keyname, server_key, nonce); - if (cmd.get_version() < protocol_version) - protocol_version = cmd.get_version(); return process_hello_cmd(server_keyname, server_key, nonce); } break; @@ -2996,12 +2992,19 @@ "anonymous netcmd received in source or source/sink role"); { protocol_role role; - string pattern; - id nonce2; - cmd.read_anonymous_cmd(role, pattern, nonce2); - if (cmd.get_version() < protocol_version) - protocol_version = cmd.get_version(); - return process_anonymous_cmd(role, pattern, nonce2); + utf8 their_include_pattern, their_exclude_pattern; + rsa_oaep_sha_data hmac_key_encrypted; + cmd.read_anonymous_cmd(role, their_include_pattern, their_exclude_pattern, hmac_key_encrypted); + L(F("received 'anonymous' netcmd from client for pattern '%s' excluding '%s' " + "in %s mode\n") + % their_include_pattern % their_exclude_pattern + % (role == source_and_sink_role ? "source and sink" : + (role == source_role ? "source " : "sink"))); + + if (!process_anonymous_cmd(role, their_include_pattern, their_exclude_pattern)) + return false; + respond_to_auth_cmd(hmac_key_encrypted); + return true; } break; @@ -3010,13 +3013,30 @@ require(voice == server_voice, "auth netcmd received in server voice"); { protocol_role role; - string pattern, signature; + string signature; + utf8 their_include_pattern, their_exclude_pattern; id client, nonce1, nonce2; - cmd.read_auth_cmd(role, pattern, client, nonce1, nonce2, signature); - if (cmd.get_version() < protocol_version) - protocol_version = cmd.get_version(); - return process_auth_cmd(role, pattern, client, - nonce1, nonce2, signature); + rsa_oaep_sha_data hmac_key_encrypted; + cmd.read_auth_cmd(role, their_include_pattern, their_exclude_pattern, + client, nonce1, hmac_key_encrypted, signature); + + hexenc their_key_hash; + encode_hexenc(client, their_key_hash); + hexenc hnonce1; + encode_hexenc(nonce1, hnonce1); + + L(F("received 'auth(hmac)' netcmd from client '%s' for pattern '%s' " + "exclude '%s' in %s mode with nonce1 '%s'\n") + % their_key_hash % their_include_pattern % their_exclude_pattern + % (role == source_and_sink_role ? "source and sink" : + (role == source_role ? "source " : "sink")) + % hnonce1); + + if (!process_auth_cmd(role, their_include_pattern, their_exclude_pattern, + client, nonce1, signature)) + return false; + respond_to_auth_cmd(hmac_key_encrypted); + return true; } break; @@ -3025,8 +3045,10 @@ require(voice == client_voice, "confirm netcmd received in client voice"); { string signature; - cmd.read_confirm_cmd(signature); - return process_confirm_cmd(signature); + cmd.read_confirm_cmd(); + this->authenticated = true; + respond_to_confirm_cmd(); + return true; } break; @@ -3148,9 +3170,11 @@ { if (!armed) { - if (cmd.read(inbuf)) + if (outbuf_size > constants::bufsz * 10) + return false; // don't pack the buffer unnecessarily + + if (cmd.read(inbuf, read_hmac)) { -// inbuf.erase(0, cmd.encoded_size()); armed = true; } } @@ -3176,7 +3200,7 @@ } catch (bad_decode & bd) { - W(F("caught bad_decode exception processing peer %s: '%s'\n") % peer_id % bd.what); + W(F("protocol error while processing peer %s: '%s'\n") % peer_id % bd.what); return false; } } @@ -3184,7 +3208,8 @@ static void call_server(protocol_role role, - vector const & patterns, + utf8 const & include_pattern, + utf8 const & exclude_pattern, app_state & app, utf8 const & address, Netxx::port_type default_port, @@ -3197,8 +3222,8 @@ P(F("connecting to %s\n") % address()); Netxx::Stream server(address().c_str(), default_port, timeout); - session sess(role, client_voice, patterns, app, - address(), server.get_socketfd(), timeout); + session sess(role, client_voice, include_pattern, exclude_pattern, + app, address(), server.get_socketfd(), timeout); while (true) { @@ -3209,7 +3234,7 @@ } catch (bad_decode & bd) { - W(F("caught bad_decode exception decoding input from peer %s: '%s'\n") + W(F("protocol error while processing peer %s: '%s'\n") % sess.peer_id % bd.what); return; } @@ -3236,7 +3261,7 @@ } catch (bad_decode & bd) { - W(F("caught bad_decode exception decoding input from peer %s: '%s'\n") + W(F("protocol error while processing peer %s: '%s'\n") % sess.peer_id % bd.what); return; } @@ -3310,7 +3335,7 @@ } catch (bad_decode & bd) { - W(F("caught bad_decode exception decoding input from peer %s: '%s', marking as bad\n") + W(F("protocol error while processing peer %s: '%s', marking as bad\n") % i->second->peer_id % bd.what); arm_failed.insert(i->first); } @@ -3327,7 +3352,8 @@ Netxx::StreamServer & server, Netxx::Timeout & timeout, protocol_role role, - vector const & patterns, + utf8 const & include_pattern, + utf8 const & exclude_pattern, map > & sessions, app_state & app) { @@ -3342,7 +3368,8 @@ else { P(F("accepted new client connection from %s\n") % client); - shared_ptr sess(new session(role, server_voice, patterns, + shared_ptr sess(new session(role, server_voice, + include_pattern, exclude_pattern, app, lexical_cast(client), client.get_socketfd(), timeout)); @@ -3367,7 +3394,7 @@ } catch (bad_decode & bd) { - W(F("caught bad_decode exception decoding input from peer %s: '%s', disconnecting\n") + W(F("protocol error while processing peer %s: '%s', disconnecting\n") % sess->peer_id % bd.what); sessions.erase(fd); live_p = false; @@ -3457,7 +3484,8 @@ static void serve_connections(protocol_role role, - vector const & patterns, + utf8 const & include_pattern, + utf8 const & exclude_pattern, app_state & app, utf8 const & address, Netxx::port_type default_port, @@ -3510,7 +3538,7 @@ // we either got a new connection else if (fd == server) handle_new_connection(addr, server, timeout, role, - patterns, sessions, app); + include_pattern, exclude_pattern, sessions, app); // or an existing session woke up else @@ -3709,7 +3737,8 @@ run_netsync_protocol(protocol_voice voice, protocol_role role, utf8 const & addr, - vector patterns, + utf8 const & include_pattern, + utf8 const & exclude_pattern, app_state & app) { try @@ -3717,7 +3746,7 @@ start_platform_netsync(); if (voice == server_voice) { - serve_connections(role, patterns, app, + serve_connections(role, include_pattern, exclude_pattern, app, addr, static_cast(constants::netsync_default_port), static_cast(constants::netsync_timeout_seconds), static_cast(constants::netsync_connection_limit)); @@ -3726,7 +3755,7 @@ { I(voice == client_voice); transaction_guard guard(app.db); - call_server(role, patterns, app, + call_server(role, include_pattern, exclude_pattern, app, addr, static_cast(constants::netsync_default_port), static_cast(constants::netsync_timeout_seconds)); guard.commit(); --- netsync.hh +++ netsync.hh @@ -22,7 +22,8 @@ void run_netsync_protocol(protocol_voice voice, protocol_role role, utf8 const & addr, - std::vector patterns, + utf8 const & include_pattern, + utf8 const & exclude_pattern, app_state & app); #endif // __NETSYNC_H__ --- tests/t_netsync_defaults.at +++ tests/t_netsync_defaults.at @@ -22,21 +22,21 @@ # First make sure netsync with explicit server/pattern override defaults AT_CHECK(MONOTONE2 set database default-server nonsense, [], [ignore], [ignore]) -AT_CHECK(MONOTONE2 set database default-pattern nonsense, [], [ignore], [ignore]) +AT_CHECK(MONOTONE2 set database default-include-pattern nonsense, [], [ignore], [ignore]) NETSYNC_CLIENT_RUN(pull, testbranch, 0) AT_CHECK(MONOTONE2 checkout --branch=testbranch --revision=$TESTBRANCH_R testdir1, [], [ignore], [ignore]) AT_CHECK(test -f testdir1/testfile) # Now make sure explicit server with default pattern works... AT_CHECK(MONOTONE2 set database default-server nonsense, [], [ignore], [ignore]) -AT_CHECK(MONOTONE2 set database default-pattern otherbranch, [], [ignore], [ignore]) +AT_CHECK(MONOTONE2 set database default-include-pattern otherbranch, [], [ignore], [ignore]) NETSYNC_CLIENT_RUN(pull, []) AT_CHECK(MONOTONE2 checkout --branch=otherbranch --revision=$OTHERBRANCH_R testdir2, [], [ignore], [ignore]) AT_CHECK(test -f testdir2/testfile) -# And finally, +# And finally, make sure that passing nothing at all also works (uses default) AT_CHECK(MONOTONE2 set database default-server 127.0.0.1:$_PORT, [], [ignore], [ignore]) -AT_CHECK(MONOTONE2 set database default-pattern thirdbranch, [], [ignore], [ignore]) +AT_CHECK(MONOTONE2 set database default-include-pattern thirdbranch, [], [ignore], [ignore]) AT_CHECK(MONOTONE2 sync, [], [ignore], [ignore]) AT_CHECK(MONOTONE2 checkout --branch=thirdbranch --revision=$THIRDBRANCH_R testdir3, [], [ignore], [ignore]) AT_CHECK(test -f testdir3/testfile) --- tests/t_netsync_globs.at +++ tests/t_netsync_globs.at @@ -0,0 +1,40 @@ +AT_SETUP([netsync with globs]) +MONOTONE_SETUP +NETSYNC_SETUP + +ADD_FILE(testfile, [foo +]) +COMMIT(1branch1) +REV11=`BASE_REVISION` + +AT_CHECK(rm -rf MT) +AT_CHECK(MONOTONE setup .) +ADD_FILE(testfile, [bar +]) +COMMIT(1branch2) +REV12=`BASE_REVISION` + +AT_CHECK(rm -rf MT) +AT_CHECK(MONOTONE setup .) +ADD_FILE(testfile, [baz +]) +COMMIT(2branch1) +REV21=`BASE_REVISION` + +NETSYNC_SERVE_START("*") + +# check a glob +NETSYNC_CLIENT_N_RUN(2, pull, "*anch2") +AT_CHECK(MONOTONE2 cat revision $REV12, [], [ignore], [ignore]) +AT_CHECK(MONOTONE2 cat revision $REV11, [1], [ignore], [ignore]) +AT_CHECK(MONOTONE2 cat revision $REV21, [1], [ignore], [ignore]) + +# check explicit multiple branches +NETSYNC_CLIENT_N_RUN(3, pull, 1branch1 2branch1) +AT_CHECK(MONOTONE3 cat revision $REV12, [1], [ignore], [ignore]) +AT_CHECK(MONOTONE3 cat revision $REV11, [], [ignore], [ignore]) +AT_CHECK(MONOTONE3 cat revision $REV21, [], [ignore], [ignore]) + +NETSYNC_SERVE_STOP + +AT_CLEANUP --- tests/t_netsync_permissions.at +++ tests/t_netsync_permissions.at @@ -18,7 +18,7 @@ return true end -function get_netsync_write_permitted(pattern, identity) +function get_netsync_write_permitted(identity) return true end @@ -106,7 +106,7 @@ return false end -function get_netsync_write_permitted(pattern, identity) +function get_netsync_write_permitted(identity) if (identity == "address@hidden") then return true end return false end --- tests/t_netsync_single.at +++ tests/t_netsync_single.at @@ -11,7 +11,7 @@ return true end -function get_netsync_write_permitted(pattern, identity) +function get_netsync_write_permitted(identity) return true end ]) --- testsuite.at +++ testsuite.at @@ -327,7 +327,7 @@ return true end -function get_netsync_write_permitted(pattern, identity) +function get_netsync_write_permitted(identity) return true end ]) @@ -664,3 +664,4 @@ m4_include(tests/t_lua_includedir.at) m4_include(tests/t_existsonpath.at) m4_include(tests/t_db_kill_branch_locally.at) +m4_include(tests/t_netsync_globs.at) --- transforms.cc +++ transforms.cc @@ -870,142 +870,8 @@ dst += linesep_str; } -// glob_to_regexp converts a sh file glob to a regexp. The regexp should -// be usable by the Boost regexp library. -// -// Pattern tranformation: -// -// - Any character except those described below are copied as they are. -// - The backslash (\) escapes the following character. The escaping -// backslash is copied to the regexp along with the following character. -// - * is transformed to .* in the regexp. -// - ? is transformed to . in the regexp. -// - { is transformed to ( in the regexp, unless within [ and ]. -// - } is transformed to ) in the regexp, unless within [ and ]. -// - , is transformed to | in the regexp, if within { and } and not -// within [ and ]. -// - ^ is escaped unless it comes directly after an unescaped [. -// - ! is transformed to ^ in the regexp if it comes directly after an -// unescaped [. -// - ] directly following an unescaped [ is escaped. -string glob_to_regexp(const string & glob) -{ - int in_braces = 0; // counter for levels if {} - bool in_brackets = false; // flags if we're inside a [], which - // has higher precedence than {}. - // Also, [ is accepted inside [] unescaped. - bool this_was_opening_bracket = false; - string tmp; - tmp.reserve(glob.size() * 2); - #ifdef BUILD_UNIT_TESTS - cerr << "DEBUG[glob_to_regexp]: input = \"" << glob << "\"" << endl; -#endif - - for (string::const_iterator i = glob.begin(); i != glob.end(); ++i) - { - char c = *i; - bool last_was_opening_bracket = this_was_opening_bracket; - this_was_opening_bracket = false; - - // Special case ^ and ! at the beginning of a [] expression. - if (in_brackets && last_was_opening_bracket - && (c == '!' || c == '^')) - { - tmp += '^'; - if (++i == glob.end()) - break; - c = *i; - } - - if (c == '\\') - { - tmp += c; - if (++i == glob.end()) - break; - tmp += *i; - } - else if (in_brackets) - { - switch(c) - { - case ']': - if (!last_was_opening_bracket) - { - in_brackets = false; - tmp += c; - break; - } - // Trickling through to the standard character conversion, - // because ] as the first character of a set is regarded as - // a normal character. - default: - if (!(isalnum(c) || c == '_')) - { - tmp += '\\'; - } - tmp += c; - break; - } - } - else - { - switch(c) - { - case '*': - tmp += ".*"; - break; - case '?': - tmp += '.'; - break; - case '{': - in_braces++; - tmp += '('; - break; - case '}': - N(in_braces != 0, - F("trying to end a brace expression in a glob when none is started")); - tmp += ')'; - in_braces--; - break; - case '[': - in_brackets = true; - this_was_opening_bracket = true; - tmp += c; - break; - case ',': - if (in_braces > 0) - { - tmp += '|'; - break; - } - // Trickling through to default: here, since a comma outside of - // brace notation is just a normal character. - default: - if (!(isalnum(c) || c == '_')) - { - tmp += '\\'; - } - tmp += c; - break; - } - } - } - - N(!in_brackets, - F("run-away bracket expression in glob")); - N(in_braces == 0, - F("run-away brace expression in glob")); - -#ifdef BUILD_UNIT_TESTS - cerr << "DEBUG[glob_to_regexp]: output = \"" << tmp << "\"" << endl; -#endif - - return tmp; -} - -#ifdef BUILD_UNIT_TESTS #include "unit_tests.hh" static void @@ -1283,15 +1149,6 @@ check_idna_encoding(); } -static void glob_to_regexp_test() -{ - BOOST_CHECK(glob_to_regexp("abc,v") == "abc\\,v"); - BOOST_CHECK(glob_to_regexp("foo[12m,]") == "foo[12m\\,]"); - // A full fledged, use all damn features test... - BOOST_CHECK(glob_to_regexp("foo.{bar*,cookie?{haha,hehe[^\\123!,]}}[!]a^b]") - == "foo\\.(bar.*|cookie.(haha|hehe[^\\123\\!\\,]))[^\\]a\\^b]"); -} - void add_transform_tests(test_suite * suite) { @@ -1303,7 +1160,6 @@ suite->add(BOOST_TEST_CASE(&join_lines_test)); suite->add(BOOST_TEST_CASE(&strip_ws_test)); suite->add(BOOST_TEST_CASE(&encode_test)); - suite->add(BOOST_TEST_CASE(&glob_to_regexp_test)); } #endif // BUILD_UNIT_TESTS --- transforms.hh +++ transforms.hh @@ -73,6 +73,10 @@ void decode_gzip(gzip const & in, T & out) { out = xform(in()); } +// string variant for netsync +template +void encode_gzip(std::string const & in, gzip & out) +{ out = xform(in); } // both at once (this is relatively common) @@ -189,6 +193,4 @@ // line-ending conversion void line_end_convert(std::string const & linesep, std::string const & src, std::string & dst); - - #endif // __TRANSFORMS_HH__ --- unit_tests.cc +++ unit_tests.cc @@ -69,9 +69,11 @@ if (t.empty() || t.find("netcmd") != t.end()) add_netcmd_tests(suite); - if (t.empty() || t.find("netcmd") != t.end()) + if (t.empty() || t.find("path_component") != t.end()) add_path_component_tests(suite); + if (t.empty() || t.find("globish") != t.end()) + add_globish_tests(suite); // all done, add our clean-shutdown-indicator suite->add(BOOST_TEST_CASE(&clean_shutdown_dummy_test)); --- unit_tests.hh +++ unit_tests.hh @@ -32,5 +32,6 @@ void add_packet_tests(test_suite * suite); void add_netcmd_tests(test_suite * suite); void add_path_component_tests(test_suite * suite); +void add_globish_tests(test_suite * suite); #endif --- vocab.cc +++ vocab.cc @@ -102,7 +102,43 @@ val.ok = true; } +inline void +verify(netsync_session_key & val) +{ + if (val.ok) + return; + if (val().size() == 0) + { + val.s.append(constants::netsync_session_key_length_in_bytes, 0); + return; + } + + N(val().size() == constants::netsync_session_key_length_in_bytes, + F("Invalid key length of %d bytes") % val().length()); + + val.ok = true; +} + +inline void +verify(netsync_hmac_value & val) +{ + if (val.ok) + return; + + if (val().size() == 0) + { + val.s.append(constants::netsync_hmac_value_length_in_bytes, 0); + return; + } + + N(val().size() == constants::netsync_hmac_value_length_in_bytes, + F("Invalid hmac length of %d bytes") % val().length()); + + val.ok = true; +} + + inline void verify(local_path & val) { --- vocab_terms.hh +++ vocab_terms.hh @@ -34,7 +34,11 @@ ATOMIC_NOVERIFY(rsa_pub_key); // some nice numbers ATOMIC_NOVERIFY(rsa_priv_key); // some nice numbers ATOMIC_NOVERIFY(rsa_sha1_signature); // some other nice numbers +ATOMIC_NOVERIFY(rsa_oaep_sha_data); +ATOMIC(netsync_session_key); // key for netsync session HMAC +ATOMIC(netsync_hmac_value); // 160-bit SHA-1 HMAC + DECORATE(revision); // thing associated with a revision DECORATE(manifest); // thing associated with a manifest DECORATE(file); // thing associated with a file