>From 30b09eb7dbfbb9423efb6bd4009203f887d90786 Mon Sep 17 00:00:00 2001 From: Matthew White Date: Tue, 30 Aug 2016 09:26:41 +0200 Subject: [PATCH 23/25] New: --trust-server-names saves Metalink/HTTP xml files using the "name" field * src/metalink.c (retrieve_from_metalink): If opt.trustservernames is true, use the basename of the metaurl's name to save the xml file * doc/metalink-standard.txt: Update doc. With --trust-server-names any Metalink/HTTP Link application/metalink4+xml file is saved using the basename of the "name" field, if any. Update Metalink/HTTP examples * testenv/Makefile.am: Add new file * testenv/Test-metalink-http-xml-trust-name.py: New file. Metalink/HTTP automated Metalink/XML, save xml files using the "name" field tests --- doc/metalink-standard.txt | 10 + src/metalink.c | 2 +- testenv/Makefile.am | 1 + testenv/Test-metalink-http-xml-trust-name.py | 273 +++++++++++++++++++++++++++ 4 files changed, 285 insertions(+), 1 deletion(-) create mode 100755 testenv/Test-metalink-http-xml-trust-name.py diff --git a/doc/metalink-standard.txt b/doc/metalink-standard.txt index 78709fb..4836a85 100644 --- a/doc/metalink-standard.txt +++ b/doc/metalink-standard.txt @@ -83,6 +83,9 @@ When --trust-server-names is on, the metalink:file "name" field parsed from Metalink/XML files is trusted. When no Metalink/XML is available, the mother URL is trusted. +Any Metalink/HTTP application/metalink4+xml file is saved using the +basename of its own Link header "name" field, if available. + 3.1.2 The final name ==================== @@ -183,6 +186,13 @@ type="application/pgp-signature" Digest: SHA-256=MWVkMWQxYTRiMzk5MDQ0MzI3NGU5NDEyZTk5OWY1ZGFmNzgyZTJlO DYzYjRjYzFhOTlmNTQwYzI2M2QwM2U2MQ== +See [2 #section-4]. + +Link: ; rel=describedby; +type="application/x-bittorrent"; name="differentname.ext" +Link: ; rel=describedby; +type="application/metalink4+xml" + 5.4 Saving files ================ diff --git a/src/metalink.c b/src/metalink.c index 1506825..21fef67 100644 --- a/src/metalink.c +++ b/src/metalink.c @@ -240,7 +240,7 @@ retrieve_from_metalink (const metalink_t* metalink) metafile = xstrdup (safename); if (opt.trustservernames) - replace_metalink_basename (&metafile, murl->url); + replace_metalink_basename (&metafile, murl->name ? murl->name : murl->url); else append_suffix_number (&metafile, ".meta#", meta_count); diff --git a/testenv/Makefile.am b/testenv/Makefile.am index 2cd5ff5..0ef0915 100644 --- a/testenv/Makefile.am +++ b/testenv/Makefile.am @@ -32,6 +32,7 @@ if METALINK_IS_ENABLED Test-metalink-http-baddigest.py \ Test-metalink-http-xml.py \ Test-metalink-http-xml-trust.py \ + Test-metalink-http-xml-trust-name.py \ Test-metalink-xml.py \ Test-metalink-xml-relpath.py \ Test-metalink-xml-abspath.py \ diff --git a/testenv/Test-metalink-http-xml-trust-name.py b/testenv/Test-metalink-http-xml-trust-name.py new file mode 100755 index 0000000..69e633e --- /dev/null +++ b/testenv/Test-metalink-http-xml-trust-name.py @@ -0,0 +1,273 @@ +#!/usr/bin/env python3 +from sys import exit +from test.http_test import HTTPTest +from misc.wget_file import WgetFile +import re +import hashlib +from base64 import b64encode + +""" + This is to test Metalink/HTTP with Metalink/XML Link headers. + + With --trust-server-names, trust the metalink:file names. + + Without --trust-server-names, don't trust the metalink:file names: + use the basename of --input-metalink, and add a sequential number + (e.g. .#1, .#2, etc.). + + Strip the directory from unsafe paths. +""" + +############# File Definitions ############################################### +bad = "Ouch!" +bad_sha256 = hashlib.sha256 (bad.encode ('UTF-8')).hexdigest () + +File1 = "Would you like some Tea?" +File1_lowPref = "Do not take this" +File1_sha256 = hashlib.sha256 (File1.encode ('UTF-8')).hexdigest () + +File2 = "This is gonna be good" +File2_lowPref = "Not this one too" +File2_sha256 = hashlib.sha256 (File2.encode ('UTF-8')).hexdigest () + +File3 = "A little more, please" +File3_lowPref = "That's just too much" +File3_sha256 = hashlib.sha256 (File3.encode ('UTF-8')).hexdigest () + +File4 = "Maybe a biscuit?" +File4_lowPref = "No, thanks" +File4_sha256 = hashlib.sha256 (File4.encode ('UTF-8')).hexdigest () + +File5 = "More Tea...?" +File5_lowPref = "I have to go..." +File5_sha256 = hashlib.sha256 (File5.encode ('UTF-8')).hexdigest () + +MetaXml1 = \ +""" + + + GNU Wget + + + GNU GPL + http://www.gnu.org/licenses/gpl.html + + Wget Test Files + 1.2.3 + Wget Test Files description + + + + {{FILE1_HASH}} + + + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 + http://{{SRV_HOST}}:{{SRV_PORT}}/File1_lowPref + http://{{SRV_HOST}}:{{SRV_PORT}}/File1 + + + + + {{FILE2_HASH}} + + + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 + http://{{SRV_HOST}}:{{SRV_PORT}}/File2_lowPref + http://{{SRV_HOST}}:{{SRV_PORT}}/File2 + + + + + {{FILE3_HASH}} + + + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 + http://{{SRV_HOST}}:{{SRV_PORT}}/File3_lowPref + http://{{SRV_HOST}}:{{SRV_PORT}}/File3 + + + + + {{FILE4_HASH}} + + + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 + http://{{SRV_HOST}}:{{SRV_PORT}}/File4_lowPref + http://{{SRV_HOST}}:{{SRV_PORT}}/File4 + + + + + {{FILE5_HASH}} + + + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 + http://{{SRV_HOST}}:{{SRV_PORT}}/File5_lowPref + http://{{SRV_HOST}}:{{SRV_PORT}}/File5 + + + + +""" + +MetaXml2 = \ +""" + + + GNU Wget + + + GNU GPL + http://www.gnu.org/licenses/gpl.html + + Wget Test Files + 1.2.3 + Wget Test Files description + + + + {{BAD_HASH}} + + + http://{{SRV_HOST}}:{{SRV_PORT}}/wrong_file + http://{{SRV_HOST}}:{{SRV_PORT}}/404 + http://{{SRV_HOST}}:{{SRV_PORT}}/bad + + + + +""" + +LinkHeaders = [ + # This file has the lowest priority, and should go last + "; rel=describedby; pri=2; type=\"application/metalink4+xml\"; name=\"newname.metalink\"", + # This file has the highest priority, and should go first + "; rel=describedby; pri=1; type=\"application/metalink4+xml\"" +] + +# This will be filled as soon as we know server hostname and port +MetaHTTPRules = {'SendHeader' : {}} + +MetaHTTP = WgetFile ("main.metalink", rules=MetaHTTPRules) + +wrong_file = WgetFile ("wrong_file", bad) + +File1_orig = WgetFile ("File1", File1) +File1_down = WgetFile ("dir/File1", File1) +File1_nono = WgetFile ("File1_lowPref", File1_lowPref) + +File2_orig = WgetFile ("File2", File2) +File2_down = WgetFile ("dir/File2", File2) +File2_nono = WgetFile ("File2_lowPref", File2_lowPref) + +# rejected by libmetalink +File3_orig = WgetFile ("File3", File3) +File3_nono = WgetFile ("File3_lowPref", File3_lowPref) + +File4_orig = WgetFile ("File4", File4) +File4_down = WgetFile ("dir/File4", File4) +File4_nono = WgetFile ("File4_lowPref", File4_lowPref) + +File5_orig = WgetFile ("File5", File5) +File5_down = WgetFile ("dir/File5", File5) +File5_nono = WgetFile ("File5_lowPref", File5_lowPref) + +MetaFile1 = WgetFile ("test1.meta4", MetaXml1) +MetaFile1_down = WgetFile ("newname.metalink", MetaXml1) + +MetaFile2 = WgetFile ("test2.meta4", MetaXml2) + +WGET_OPTIONS = "--trust-server-names --metalink-over-http --metalink-index=2" +WGET_URLS = [["main.metalink"]] + +RequestList = [[ + "HEAD /main.metalink", + "GET /404", + "GET /wrong_file", + "GET /test1.meta4", + "GET /File1", + "GET /File2", + "GET /File4", + "GET /File5" +]] + +Files = [[ + MetaHTTP, + wrong_file, + MetaFile1, MetaFile2, + File1_orig, File1_nono, + File2_orig, File2_nono, + File3_orig, File3_nono, + File4_orig, File4_nono, + File5_orig, File5_nono +]] +Existing_Files = [] + +ExpectedReturnCode = 0 +ExpectedDownloadedFiles = [ + MetaFile1_down, + File1_down, + File2_down, + File4_down, + File5_down +] + +################ Pre and Post Test Hooks ##################################### +pre_test = { + "ServerFiles" : Files, + "LocalFiles" : Existing_Files +} +test_options = { + "WgetCommands" : WGET_OPTIONS, + "Urls" : WGET_URLS +} +post_test = { + "ExpectedFiles" : ExpectedDownloadedFiles, + "ExpectedRetcode" : ExpectedReturnCode, + "FilesCrawled" : RequestList +} + +http_test = HTTPTest ( + pre_hook=pre_test, + test_params=test_options, + post_hook=post_test +) + +http_test.server_setup() +### Get and use dynamic server sockname +srv_host, srv_port = http_test.servers[0].server_inst.socket.getsockname () + +MetaXml1 = re.sub (r'{{FILE1_HASH}}', File1_sha256, MetaXml1) +MetaXml1 = re.sub (r'{{FILE2_HASH}}', File2_sha256, MetaXml1) +MetaXml1 = re.sub (r'{{FILE3_HASH}}', File3_sha256, MetaXml1) +MetaXml1 = re.sub (r'{{FILE4_HASH}}', File4_sha256, MetaXml1) +MetaXml1 = re.sub (r'{{FILE5_HASH}}', File5_sha256, MetaXml1) +MetaXml1 = re.sub (r'{{SRV_HOST}}', srv_host, MetaXml1) +MetaXml1 = re.sub (r'{{SRV_PORT}}', str (srv_port), MetaXml1) +MetaFile1.content = MetaXml1 +MetaFile1_down.content = MetaXml1 + +MetaXml2 = re.sub (r'{{BAD_HASH}}', bad_sha256, MetaXml2) +MetaXml2 = re.sub (r'{{SRV_HOST}}', srv_host, MetaXml2) +MetaXml2 = re.sub (r'{{SRV_PORT}}', str (srv_port), MetaXml2) +MetaFile2.content = MetaXml2 + +# Helper function for hostname, port and digest substitution +def SubstituteServerInfo (text, host, port): + text = re.sub (r'{{SRV_HOST}}', host, text) + text = re.sub (r'{{SRV_PORT}}', str (port), text) + return text + +MetaHTTPRules["SendHeader"] = { + 'Link': [ SubstituteServerInfo (LinkHeader, srv_host, srv_port) + for LinkHeader in LinkHeaders ] +} + +err = http_test.begin () + +exit (err) -- 2.7.3