From 3468a19a2802b8f52a0559f1839b010b979def05 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sat, 18 Oct 2014 02:40:07 -0700 Subject: [PATCH 001/453] work in progress --- multipart/.gitignore | 3 + multipart/Cargo.toml | 8 + multipart/mime_types.json | 563 ++++++++++++++++++++++++++++++++++++++ multipart/src/client.rs | 144 ++++++++++ multipart/src/lib.rs | 17 ++ multipart/src/server.rs | 0 6 files changed, 735 insertions(+) create mode 100644 multipart/.gitignore create mode 100644 multipart/Cargo.toml create mode 100644 multipart/mime_types.json create mode 100644 multipart/src/client.rs create mode 100644 multipart/src/lib.rs create mode 100644 multipart/src/server.rs diff --git a/multipart/.gitignore b/multipart/.gitignore new file mode 100644 index 000000000..1a1e47390 --- /dev/null +++ b/multipart/.gitignore @@ -0,0 +1,3 @@ +/target +/Cargo.lock +*.swp diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml new file mode 100644 index 000000000..111eb7a31 --- /dev/null +++ b/multipart/Cargo.toml @@ -0,0 +1,8 @@ +[package] + +name = "multipart" +version = "0.0.1" +authors = ["Austin Bonander "] + +[dependencies.hyper] +git = "https://github.com/hyperium/hyper" diff --git a/multipart/mime_types.json b/multipart/mime_types.json new file mode 100644 index 000000000..252b279da --- /dev/null +++ b/multipart/mime_types.json @@ -0,0 +1,563 @@ +{ + "_source":"https://rawgithubusercontent.com/samuelneff/MimeTypeMap/master/src/MimeTypeMap.cs", + "323": "text/h323", + "3g2": "video/3gpp2", + "3gp": "video/3gpp", + "3gp2": "video/3gpp2", + "3gpp": "video/3gpp", + "7z": "application/x-7z-compressed", + "aa": "audio/audible", + "AAC": "audio/aac", + "aaf": "application/octet-stream", + "aax": "audio/vnd.audible.aax", + "ac3": "audio/ac3", + "aca": "application/octet-stream", + "accda": "application/msaccess.addin", + "accdb": "application/msaccess", + "accdc": "application/msaccess.cab", + "accde": "application/msaccess", + "accdr": "application/msaccess.runtime", + "accdt": "application/msaccess", + "accdw": "application/msaccess.webapplication", + "accft": "application/msaccess.ftemplate", + "acx": "application/internet-property-stream", + "AddIn": "text/xml", + "ade": "application/msaccess", + "adobebridge": "application/x-bridge-url", + "adp": "application/msaccess", + "ADT": "audio/vnd.dlna.adts", + "ADTS": "audio/aac", + "afm": "application/octet-stream", + "ai": "application/postscript", + "aif": "audio/x-aiff", + "aifc": "audio/aiff", + "aiff": "audio/aiff", + "air": "application/vnd.adobe.air-application-installer-package+zip", + "amc": "application/x-mpeg", + "application": "application/x-ms-application", + "art": "image/x-jg", + "asa": "application/xml", + "asax": "application/xml", + "ascx": "application/xml", + "asd": "application/octet-stream", + "asf": "video/x-ms-asf", + "ashx": "application/xml", + "asi": "application/octet-stream", + "asm": "text/plain", + "asmx": "application/xml", + "aspx": "application/xml", + "asr": "video/x-ms-asf", + "asx": "video/x-ms-asf", + "atom": "application/atom+xml", + "au": "audio/basic", + "avi": "video/x-msvideo", + "axs": "application/olescript", + "bas": "text/plain", + "bcpio": "application/x-bcpio", + "bin": "application/octet-stream", + "bmp": "image/bmp", + "c": "text/plain", + "cab": "application/octet-stream", + "caf": "audio/x-caf", + "calx": "application/vnd.ms-office.calx", + "cat": "application/vnd.ms-pki.seccat", + "cc": "text/plain", + "cd": "text/plain", + "cdda": "audio/aiff", + "cdf": "application/x-cdf", + "cer": "application/x-x509-ca-cert", + "chm": "application/octet-stream", + "class": "application/x-java-applet", + "clp": "application/x-msclip", + "cmx": "image/x-cmx", + "cnf": "text/plain", + "cod": "image/cis-cod", + "config": "application/xml", + "contact": "text/x-ms-contact", + "coverage": "application/xml", + "cpio": "application/x-cpio", + "cpp": "text/plain", + "crd": "application/x-mscardfile", + "crl": "application/pkix-crl", + "crt": "application/x-x509-ca-cert", + "cs": "text/plain", + "csdproj": "text/plain", + "csh": "application/x-csh", + "csproj": "text/plain", + "css": "text/css", + "csv": "text/csv", + "cur": "application/octet-stream", + "cxx": "text/plain", + "dat": "application/octet-stream", + "datasource": "application/xml", + "dbproj": "text/plain", + "dcr": "application/x-director", + "def": "text/plain", + "deploy": "application/octet-stream", + "der": "application/x-x509-ca-cert", + "dgml": "application/xml", + "dib": "image/bmp", + "dif": "video/x-dv", + "dir": "application/x-director", + "disco": "text/xml", + "dll": "application/x-msdownload", + "dll.config": "text/xml", + "dlm": "text/dlm", + "doc": "application/msword", + "docm": "application/vnd.ms-word.document.macroEnabled.12", + "docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + "dot": "application/msword", + "dotm": "application/vnd.ms-word.template.macroEnabled.12", + "dotx": "application/vnd.openxmlformats-officedocument.wordprocessingml.template", + "dsp": "application/octet-stream", + "dsw": "text/plain", + "dtd": "text/xml", + "dtsConfig": "text/xml", + "dv": "video/x-dv", + "dvi": "application/x-dvi", + "dwf": "drawing/x-dwf", + "dwp": "application/octet-stream", + "dxr": "application/x-director", + "eml": "message/rfc822", + "emz": "application/octet-stream", + "eot": "application/octet-stream", + "eps": "application/postscript", + "etl": "application/etl", + "etx": "text/x-setext", + "evy": "application/envoy", + "exe": "application/octet-stream", + "exe.config": "text/xml", + "fdf": "application/vnd.fdf", + "fif": "application/fractals", + "filters": "Application/xml", + "fla": "application/octet-stream", + "flr": "x-world/x-vrml", + "flv": "video/x-flv", + "fsscript": "application/fsharp-script", + "fsx": "application/fsharp-script", + "generictest": "application/xml", + "gif": "image/gif", + "group": "text/x-ms-group", + "gsm": "audio/x-gsm", + "gtar": "application/x-gtar", + "gz": "application/x-gzip", + "h": "text/plain", + "hdf": "application/x-hdf", + "hdml": "text/x-hdml", + "hhc": "application/x-oleobject", + "hhk": "application/octet-stream", + "hhp": "application/octet-stream", + "hlp": "application/winhlp", + "hpp": "text/plain", + "hqx": "application/mac-binhex40", + "hta": "application/hta", + "htc": "text/x-component", + "htm": "text/html", + "html": "text/html", + "htt": "text/webviewhtml", + "hxa": "application/xml", + "hxc": "application/xml", + "hxd": "application/octet-stream", + "hxe": "application/xml", + "hxf": "application/xml", + "hxh": "application/octet-stream", + "hxi": "application/octet-stream", + "hxk": "application/xml", + "hxq": "application/octet-stream", + "hxr": "application/octet-stream", + "hxs": "application/octet-stream", + "hxt": "text/html", + "hxv": "application/xml", + "hxw": "application/octet-stream", + "hxx": "text/plain", + "i": "text/plain", + "ico": "image/x-icon", + "ics": "application/octet-stream", + "idl": "text/plain", + "ief": "image/ief", + "iii": "application/x-iphone", + "inc": "text/plain", + "inf": "application/octet-stream", + "inl": "text/plain", + "ins": "application/x-internet-signup", + "ipa": "application/x-itunes-ipa", + "ipg": "application/x-itunes-ipg", + "ipproj": "text/plain", + "ipsw": "application/x-itunes-ipsw", + "iqy": "text/x-ms-iqy", + "isp": "application/x-internet-signup", + "ite": "application/x-itunes-ite", + "itlp": "application/x-itunes-itlp", + "itms": "application/x-itunes-itms", + "itpc": "application/x-itunes-itpc", + "IVF": "video/x-ivf", + "jar": "application/java-archive", + "java": "application/octet-stream", + "jck": "application/liquidmotion", + "jcz": "application/liquidmotion", + "jfif": "image/pjpeg", + "jnlp": "application/x-java-jnlp-file", + "jpb": "application/octet-stream", + "jpe": "image/jpeg", + "jpeg": "image/jpeg", + "jpg": "image/jpeg", + "js": "application/x-javascript", + "json": "application/json", + "jsx": "text/jscript", + "jsxbin": "text/plain", + "latex": "application/x-latex", + "library-ms": "application/windows-library+xml", + "lit": "application/x-ms-reader", + "loadtest": "application/xml", + "lpk": "application/octet-stream", + "lsf": "video/x-la-asf", + "lst": "text/plain", + "lsx": "video/x-la-asf", + "lzh": "application/octet-stream", + "m13": "application/x-msmediaview", + "m14": "application/x-msmediaview", + "m1v": "video/mpeg", + "m2t": "video/vnd.dlna.mpeg-tts", + "m2ts": "video/vnd.dlna.mpeg-tts", + "m2v": "video/mpeg", + "m3u": "audio/x-mpegurl", + "m3u8": "audio/x-mpegurl", + "m4a": "audio/m4a", + "m4b": "audio/m4b", + "m4p": "audio/m4p", + "m4r": "audio/x-m4r", + "m4v": "video/x-m4v", + "mac": "image/x-macpaint", + "mak": "text/plain", + "man": "application/x-troff-man", + "manifest": "application/x-ms-manifest", + "map": "text/plain", + "master": "application/xml", + "mda": "application/msaccess", + "mdb": "application/x-msaccess", + "mde": "application/msaccess", + "mdp": "application/octet-stream", + "me": "application/x-troff-me", + "mfp": "application/x-shockwave-flash", + "mht": "message/rfc822", + "mhtml": "message/rfc822", + "mid": "audio/mid", + "midi": "audio/mid", + "mix": "application/octet-stream", + "mk": "text/plain", + "mmf": "application/x-smaf", + "mno": "text/xml", + "mny": "application/x-msmoney", + "mod": "video/mpeg", + "mov": "video/quicktime", + "movie": "video/x-sgi-movie", + "mp2": "video/mpeg", + "mp2v": "video/mpeg", + "mp3": "audio/mpeg", + "mp4": "video/mp4", + "mp4v": "video/mp4", + "mpa": "video/mpeg", + "mpe": "video/mpeg", + "mpeg": "video/mpeg", + "mpf": "application/vnd.ms-mediapackage", + "mpg": "video/mpeg", + "mpp": "application/vnd.ms-project", + "mpv2": "video/mpeg", + "mqv": "video/quicktime", + "ms": "application/x-troff-ms", + "msi": "application/octet-stream", + "mso": "application/octet-stream", + "mts": "video/vnd.dlna.mpeg-tts", + "mtx": "application/xml", + "mvb": "application/x-msmediaview", + "mvc": "application/x-miva-compiled", + "mxp": "application/x-mmxp", + "nc": "application/x-netcdf", + "nsc": "video/x-ms-asf", + "nws": "message/rfc822", + "ocx": "application/octet-stream", + "oda": "application/oda", + "odc": "text/x-ms-odc", + "odh": "text/plain", + "odl": "text/plain", + "odp": "application/vnd.oasis.opendocument.presentation", + "ods": "application/oleobject", + "odt": "application/vnd.oasis.opendocument.text", + "one": "application/onenote", + "onea": "application/onenote", + "onepkg": "application/onenote", + "onetmp": "application/onenote", + "onetoc": "application/onenote", + "onetoc2": "application/onenote", + "orderedtest": "application/xml", + "osdx": "application/opensearchdescription+xml", + "p10": "application/pkcs10", + "p12": "application/x-pkcs12", + "p7b": "application/x-pkcs7-certificates", + "p7c": "application/pkcs7-mime", + "p7m": "application/pkcs7-mime", + "p7r": "application/x-pkcs7-certreqresp", + "p7s": "application/pkcs7-signature", + "pbm": "image/x-portable-bitmap", + "pcast": "application/x-podcast", + "pct": "image/pict", + "pcx": "application/octet-stream", + "pcz": "application/octet-stream", + "pdf": "application/pdf", + "pfb": "application/octet-stream", + "pfm": "application/octet-stream", + "pfx": "application/x-pkcs12", + "pgm": "image/x-portable-graymap", + "pic": "image/pict", + "pict": "image/pict", + "pkgdef": "text/plain", + "pkgundef": "text/plain", + "pko": "application/vnd.ms-pki.pko", + "pls": "audio/scpls", + "pma": "application/x-perfmon", + "pmc": "application/x-perfmon", + "pml": "application/x-perfmon", + "pmr": "application/x-perfmon", + "pmw": "application/x-perfmon", + "png": "image/png", + "pnm": "image/x-portable-anymap", + "pnt": "image/x-macpaint", + "pntg": "image/x-macpaint", + "pnz": "image/png", + "pot": "application/vnd.ms-powerpoint", + "potm": "application/vnd.ms-powerpoint.template.macroEnabled.12", + "potx": "application/vnd.openxmlformats-officedocument.presentationml.template", + "ppa": "application/vnd.ms-powerpoint", + "ppam": "application/vnd.ms-powerpoint.addin.macroEnabled.12", + "ppm": "image/x-portable-pixmap", + "pps": "application/vnd.ms-powerpoint", + "ppsm": "application/vnd.ms-powerpoint.slideshow.macroEnabled.12", + "ppsx": "application/vnd.openxmlformats-officedocument.presentationml.slideshow", + "ppt": "application/vnd.ms-powerpoint", + "pptm": "application/vnd.ms-powerpoint.presentation.macroEnabled.12", + "pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation", + "prf": "application/pics-rules", + "prm": "application/octet-stream", + "prx": "application/octet-stream", + "ps": "application/postscript", + "psc1": "application/PowerShell", + "psd": "application/octet-stream", + "psess": "application/xml", + "psm": "application/octet-stream", + "psp": "application/octet-stream", + "pub": "application/x-mspublisher", + "pwz": "application/vnd.ms-powerpoint", + "qht": "text/x-html-insertion", + "qhtm": "text/x-html-insertion", + "qt": "video/quicktime", + "qti": "image/x-quicktime", + "qtif": "image/x-quicktime", + "qtl": "application/x-quicktimeplayer", + "qxd": "application/octet-stream", + "ra": "audio/x-pn-realaudio", + "ram": "audio/x-pn-realaudio", + "rar": "application/octet-stream", + "ras": "image/x-cmu-raster", + "rat": "application/rat-file", + "rc": "text/plain", + "rc2": "text/plain", + "rct": "text/plain", + "rdlc": "application/xml", + "resx": "application/xml", + "rf": "image/vnd.rn-realflash", + "rgb": "image/x-rgb", + "rgs": "text/plain", + "rm": "application/vnd.rn-realmedia", + "rmi": "audio/mid", + "rmp": "application/vnd.rn-rn_music_package", + "roff": "application/x-troff", + "rpm": "audio/x-pn-realaudio-plugin", + "rqy": "text/x-ms-rqy", + "rtf": "application/rtf", + "rtx": "text/richtext", + "ruleset": "application/xml", + "s": "text/plain", + "safariextz": "application/x-safari-safariextz", + "scd": "application/x-msschedule", + "sct": "text/scriptlet", + "sd2": "audio/x-sd2", + "sdp": "application/sdp", + "sea": "application/octet-stream", + "searchConnector-ms": "application/windows-search-connector+xml", + "setpay": "application/set-payment-initiation", + "setreg": "application/set-registration-initiation", + "settings": "application/xml", + "sgimb": "application/x-sgimb", + "sgml": "text/sgml", + "sh": "application/x-sh", + "shar": "application/x-shar", + "shtml": "text/html", + "sit": "application/x-stuffit", + "sitemap": "application/xml", + "skin": "application/xml", + "sldm": "application/vnd.ms-powerpoint.slide.macroEnabled.12", + "sldx": "application/vnd.openxmlformats-officedocument.presentationml.slide", + "slk": "application/vnd.ms-excel", + "sln": "text/plain", + "slupkg-ms": "application/x-ms-license", + "smd": "audio/x-smd", + "smi": "application/octet-stream", + "smx": "audio/x-smd", + "smz": "audio/x-smd", + "snd": "audio/basic", + "snippet": "application/xml", + "snp": "application/octet-stream", + "sol": "text/plain", + "sor": "text/plain", + "spc": "application/x-pkcs7-certificates", + "spl": "application/futuresplash", + "src": "application/x-wais-source", + "srf": "text/plain", + "SSISDeploymentManifest": "text/xml", + "ssm": "application/streamingmedia", + "sst": "application/vnd.ms-pki.certstore", + "stl": "application/vnd.ms-pki.stl", + "sv4cpio": "application/x-sv4cpio", + "sv4crc": "application/x-sv4crc", + "svc": "application/xml", + "swf": "application/x-shockwave-flash", + "t": "application/x-troff", + "tar": "application/x-tar", + "tcl": "application/x-tcl", + "testrunconfig": "application/xml", + "testsettings": "application/xml", + "tex": "application/x-tex", + "texi": "application/x-texinfo", + "texinfo": "application/x-texinfo", + "tgz": "application/x-compressed", + "thmx": "application/vnd.ms-officetheme", + "thn": "application/octet-stream", + "tif": "image/tiff", + "tiff": "image/tiff", + "tlh": "text/plain", + "tli": "text/plain", + "toc": "application/octet-stream", + "tr": "application/x-troff", + "trm": "application/x-msterminal", + "trx": "application/xml", + "ts": "video/vnd.dlna.mpeg-tts", + "tsv": "text/tab-separated-values", + "ttf": "application/octet-stream", + "tts": "video/vnd.dlna.mpeg-tts", + "txt": "text/plain", + "u32": "application/octet-stream", + "uls": "text/iuls", + "user": "text/plain", + "ustar": "application/x-ustar", + "vb": "text/plain", + "vbdproj": "text/plain", + "vbk": "video/mpeg", + "vbproj": "text/plain", + "vbs": "text/vbscript", + "vcf": "text/x-vcard", + "vcproj": "Application/xml", + "vcs": "text/plain", + "vcxproj": "Application/xml", + "vddproj": "text/plain", + "vdp": "text/plain", + "vdproj": "text/plain", + "vdx": "application/vnd.ms-visio.viewer", + "vml": "text/xml", + "vscontent": "application/xml", + "vsct": "text/xml", + "vsd": "application/vnd.visio", + "vsi": "application/ms-vsi", + "vsix": "application/vsix", + "vsixlangpack": "text/xml", + "vsixmanifest": "text/xml", + "vsmdi": "application/xml", + "vspscc": "text/plain", + "vss": "application/vnd.visio", + "vsscc": "text/plain", + "vssettings": "text/xml", + "vssscc": "text/plain", + "vst": "application/vnd.visio", + "vstemplate": "text/xml", + "vsto": "application/x-ms-vsto", + "vsw": "application/vnd.visio", + "vsx": "application/vnd.visio", + "vtx": "application/vnd.visio", + "wav": "audio/wav", + "wave": "audio/wav", + "wax": "audio/x-ms-wax", + "wbk": "application/msword", + "wbmp": "image/vnd.wap.wbmp", + "wcm": "application/vnd.ms-works", + "wdb": "application/vnd.ms-works", + "wdp": "image/vnd.ms-photo", + "webarchive": "application/x-safari-webarchive", + "webtest": "application/xml", + "wiq": "application/xml", + "wiz": "application/msword", + "wks": "application/vnd.ms-works", + "WLMP": "application/wlmoviemaker", + "wlpginstall": "application/x-wlpg-detect", + "wlpginstall3": "application/x-wlpg3-detect", + "wm": "video/x-ms-wm", + "wma": "audio/x-ms-wma", + "wmd": "application/x-ms-wmd", + "wmf": "application/x-msmetafile", + "wml": "text/vnd.wap.wml", + "wmlc": "application/vnd.wap.wmlc", + "wmls": "text/vnd.wap.wmlscript", + "wmlsc": "application/vnd.wap.wmlscriptc", + "wmp": "video/x-ms-wmp", + "wmv": "video/x-ms-wmv", + "wmx": "video/x-ms-wmx", + "wmz": "application/x-ms-wmz", + "wpl": "application/vnd.ms-wpl", + "wps": "application/vnd.ms-works", + "wri": "application/x-mswrite", + "wrl": "x-world/x-vrml", + "wrz": "x-world/x-vrml", + "wsc": "text/scriptlet", + "wsdl": "text/xml", + "wvx": "video/x-ms-wvx", + "x": "application/directx", + "xaf": "x-world/x-vrml", + "xaml": "application/xaml+xml", + "xap": "application/x-silverlight-app", + "xbap": "application/x-ms-xbap", + "xbm": "image/x-xbitmap", + "xdr": "text/plain", + "xht": "application/xhtml+xml", + "xhtml": "application/xhtml+xml", + "xla": "application/vnd.ms-excel", + "xlam": "application/vnd.ms-excel.addin.macroEnabled.12", + "xlc": "application/vnd.ms-excel", + "xld": "application/vnd.ms-excel", + "xlk": "application/vnd.ms-excel", + "xll": "application/vnd.ms-excel", + "xlm": "application/vnd.ms-excel", + "xls": "application/vnd.ms-excel", + "xlsb": "application/vnd.ms-excel.sheet.binary.macroEnabled.12", + "xlsm": "application/vnd.ms-excel.sheet.macroEnabled.12", + "xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + "xlt": "application/vnd.ms-excel", + "xltm": "application/vnd.ms-excel.template.macroEnabled.12", + "xltx": "application/vnd.openxmlformats-officedocument.spreadsheetml.template", + "xlw": "application/vnd.ms-excel", + "xml": "text/xml", + "xmta": "application/xml", + "xof": "x-world/x-vrml", + "XOML": "text/plain", + "xpm": "image/x-xpixmap", + "xps": "application/vnd.ms-xpsdocument", + "xrm-ms": "text/xml", + "xsc": "application/xml", + "xsd": "text/xml", + "xsf": "text/xml", + "xsl": "text/xml", + "xslt": "text/xml", + "xsn": "application/octet-stream", + "xss": "application/xml", + "xtp": "application/octet-stream", + "xwd": "image/x-xwindowdump", + "z": "application/x-compress", + "zip": "application/x-zip-compressed" +} diff --git a/multipart/src/client.rs b/multipart/src/client.rs new file mode 100644 index 000000000..48636c7c1 --- /dev/null +++ b/multipart/src/client.rs @@ -0,0 +1,144 @@ +use hyper::client::{Request, Response}; +use hyper::net::{Fresh, Streaming}; +use hyper::{HttpResult, HttpIoError}; + +use mime::{mod, Mime}; + +use serialize::json; + +use std::cell::RefCell; + +use std::collections::HashMap; + +use std::io::IoResult; +use std::io::fs::File; + +const BOUNDARY_LEN: uint = 8; + +pub struct Multipart<'a> { + fields: Vec<(&'a str, MultipartField<'a>)>, +} + +impl<'a> Multipart<'a> { + + pub fn new(&self) -> Multipart { + Multipart { + fields: Vec::new(), + } + } + + pub fn add_text(&mut self, name: &'a str, val: &'a str) { + self.fields.push((name, Text(val))); + } + + /// Add the file to the multipart request, guessing its `Content-Type` from its extension + pub fn add_file(&mut self, name: &'a str, file: &'a mut File) { + let filename = file.path().filename_str().map(|s| s.into_string()); + let content_type = guess_mime_type(file.path()); + + self.fields.push((name, Stream { + filename: filename, + stream: file, + content_type: content_type, + })) + } + + /// Apply the appropriate headers to the `Request` and send the data. + pub fn send(self, mut req: Request) -> HttpResult { + self.apply_headers(&mut req); + + let mut req = try!(req.start()); + try!(self.write_request(&mut req)); + req.send() + } + + fn apply_headers(&self, req: &mut Request){ + let headers = req.headers_mut(); + + + } + + fn write_request(self, req: &mut Request) -> HttpResult<()> { + + + } +} + +/// Generate a random alphanumeric sequence of length `len` +fn random_alphanumeric(len: uint) -> String { + use std::char::is_alphanumeric; + use std::rand::{task_rng, Rng}; + + task_rng().gen_ascii_chars().filter(|c| is_alphanumeric(*c)).take(len).collect() +} + +fn io_to_http(res: IoResult) -> HttpResult { + res.map_err(|e| HttpIoError(e)) +} + +enum MultipartField<'a> { + Text(&'a str), + Stream { + filename: Option, + stream: &'a mut Reader + 'a, + content_type: Mime, + }, +} + +/// Guess the MIME type of the `Path` by its extension. +/// +/// **Guess** is the operative word here, as the contents of a file +/// may not or may not match its MIME type/extension. +pub fn guess_mime_type(path: &Path) -> Mime { + let ext = path.extension_str().unwrap_or(""); + + get_mime_type(ext) +} + +local_data_key!(mime_types_cache: RefCell>) + +/// Get the MIME type associated with a file extension +// MIME Types are cached in a task-local heap +pub fn get_mime_type(ext: &str) -> Mime { + if ext.is_empty() { return octet_stream(); } + + let ext = ext.into_string(); + + let cache = if let Some(cache) = mime_types_cache.get() { cache } + else { + mime_types_cache.replace(Some(RefCell::new(HashMap::new()))); + mime_types_cache.get().unwrap() + }; + + if let Some(mime_type) = cache.borrow().find(&ext) { + return mime_type.clone(); + } + + let mime_type = find_mime_type(&ext); + + cache.borrow_mut().insert(ext, mime_type.clone()); + + mime_type +} + +const MIME_TYPES: &'static str = include_str!("../mime_types.json"); + +/// Load the MIME_TYPES as JSON and try to locate `ext` +fn find_mime_type(ext: &String) -> Mime { + json::from_str(MIME_TYPES).unwrap() + .find(ext).and_then(|j| j.as_string()) + .and_then(from_str::) + .unwrap_or_else(octet_stream) +} + +fn octet_stream() -> Mime { + Mime(mime::Application, mime::SubExt("octet-stream".into_string()), Vec::new()) +} + +#[test] +fn test_mime_type_guessing() { + assert!(get_mime_type("gif").to_string() == "image/gif".to_string()); + assert!(get_mime_type("txt").to_string() == "text/plain".to_string()); + assert!(get_mime_type("blahblah").to_string() == "application/octet-stream".to_string()); +} + diff --git a/multipart/src/lib.rs b/multipart/src/lib.rs new file mode 100644 index 000000000..75a11ab43 --- /dev/null +++ b/multipart/src/lib.rs @@ -0,0 +1,17 @@ +#![feature(if_let, slicing_syntax, struct_variant, default_type_params)] +extern crate hyper; +extern crate mime; +extern crate serialize; + +pub mod client; +pub mod server; + +#[cfg(test)] +mod test { + + #[test] + fn client_api_test() { + + } + +} diff --git a/multipart/src/server.rs b/multipart/src/server.rs new file mode 100644 index 000000000..e69de29bb From 6f099b463d36556b328f63469f5fe908ef083b80 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Tue, 21 Oct 2014 01:23:42 -0700 Subject: [PATCH 002/453] Initial commit --- multipart/.gitignore | 11 +++++++++++ multipart/LICENSE | 22 ++++++++++++++++++++++ multipart/README.md | 4 ++++ 3 files changed, 37 insertions(+) create mode 100644 multipart/.gitignore create mode 100644 multipart/LICENSE create mode 100644 multipart/README.md diff --git a/multipart/.gitignore b/multipart/.gitignore new file mode 100644 index 000000000..37727f91c --- /dev/null +++ b/multipart/.gitignore @@ -0,0 +1,11 @@ +# Compiled files +*.o +*.so +*.rlib +*.dll + +# Executables +*.exe + +# Generated by Cargo +/target/ diff --git a/multipart/LICENSE b/multipart/LICENSE new file mode 100644 index 000000000..f0b4b9e27 --- /dev/null +++ b/multipart/LICENSE @@ -0,0 +1,22 @@ +The MIT License (MIT) + +Copyright (c) 2014 Austin Bonander + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + diff --git a/multipart/README.md b/multipart/README.md new file mode 100644 index 000000000..71cffb3d5 --- /dev/null +++ b/multipart/README.md @@ -0,0 +1,4 @@ +multipart +========= + +An extension to hyperium/hyper that adds support for HTTP multipart requests From 5f8e506ad95687933f8532c6df672e57d4a31930 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Tue, 21 Oct 2014 01:24:11 -0700 Subject: [PATCH 003/453] work in progress, client done (mostly) --- multipart/src/client.rs | 81 ++++++++++++++++++++------- multipart/src/lib.rs | 53 ++++++++++++++++-- multipart/src/server.rs | 120 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 230 insertions(+), 24 deletions(-) diff --git a/multipart/src/client.rs b/multipart/src/client.rs index 48636c7c1..0a426328c 100644 --- a/multipart/src/client.rs +++ b/multipart/src/client.rs @@ -1,4 +1,5 @@ use hyper::client::{Request, Response}; +use hyper::header::common::ContentType; use hyper::net::{Fresh, Streaming}; use hyper::{HttpResult, HttpIoError}; @@ -12,23 +13,31 @@ use std::collections::HashMap; use std::io::IoResult; use std::io::fs::File; +use std::io; + +use super::{MultipartField, TextField, FileField, FileStream, MultipartFile}; const BOUNDARY_LEN: uint = 8; pub struct Multipart<'a> { fields: Vec<(&'a str, MultipartField<'a>)>, + boundary: String, } +/// Shorthand for a writable request (`Request`) +type ReqWrite = Request; + impl<'a> Multipart<'a> { - pub fn new(&self) -> Multipart { + pub fn new() -> Multipart<'a> { Multipart { fields: Vec::new(), + boundary: random_alphanumeric(BOUNDARY_LEN), } } pub fn add_text(&mut self, name: &'a str, val: &'a str) { - self.fields.push((name, Text(val))); + self.fields.push((name, TextField(val))); } /// Add the file to the multipart request, guessing its `Content-Type` from its extension @@ -36,55 +45,87 @@ impl<'a> Multipart<'a> { let filename = file.path().filename_str().map(|s| s.into_string()); let content_type = guess_mime_type(file.path()); - self.fields.push((name, Stream { + self.fields.push((name, FileField(MultipartFile { filename: filename, - stream: file, + reader: FileStream(file), content_type: content_type, - })) + }))) } /// Apply the appropriate headers to the `Request` and send the data. pub fn send(self, mut req: Request) -> HttpResult { self.apply_headers(&mut req); + debug!("{}", req.headers()); + let mut req = try!(req.start()); - try!(self.write_request(&mut req)); + try!(io_to_http(self.write_request(&mut req))); req.send() } fn apply_headers(&self, req: &mut Request){ let headers = req.headers_mut(); - - + + headers.set(ContentType(multipart_mime(self.boundary[]))) } - fn write_request(self, req: &mut Request) -> HttpResult<()> { - - + fn write_request(self, req: &mut ReqWrite) -> IoResult<()> { + let Multipart{ fields, boundary } = self; + + try!(write_boundary(req, boundary[])); + + for (name, field) in fields.into_iter() { + try!(write!(req, "Content-Disposition: form-data; name=\"{}\"", name)); + + try!(match field { + TextField(text) => req.write(b"\r\n\r\n") + .and_then(|_| write_line(req, text)), // Style suggestions welcome + FileField(file) => write_file(req, file), + _ => unimplemented!(), + }); + + try!(write_boundary(req, boundary[])); + } + + Ok(()) } + +} + +fn write_boundary(req: &mut ReqWrite, boundary: &str) -> IoResult<()> { + write!(req, "--{}\r\n", boundary) +} + +fn write_file<'a>(req: &mut ReqWrite, mut file: MultipartFile<'a>) -> IoResult<()> { + try!(file.filename.map(|filename| write!(req, "; filename=\"{}\"\r\n", filename)).unwrap_or(Ok(()))); + try!(write!(req, "Content-Type: {}\r\n\r\n", file.content_type)); + io::util::copy(&mut file.reader, req) +} + +/// Specialized write_line that writes CRLF after a line as per W3C specs +fn write_line(req: &mut ReqWrite, s: &str) -> IoResult<()> { + req.write_str(s).and_then(|_| req.write(b"\r\n")) } /// Generate a random alphanumeric sequence of length `len` fn random_alphanumeric(len: uint) -> String { - use std::char::is_alphanumeric; use std::rand::{task_rng, Rng}; - task_rng().gen_ascii_chars().filter(|c| is_alphanumeric(*c)).take(len).collect() + task_rng().gen_ascii_chars().take(len).collect() } fn io_to_http(res: IoResult) -> HttpResult { res.map_err(|e| HttpIoError(e)) } -enum MultipartField<'a> { - Text(&'a str), - Stream { - filename: Option, - stream: &'a mut Reader + 'a, - content_type: Mime, - }, +fn multipart_mime(bound: &str) -> Mime { + mime::Mime( + mime::Multipart, mime::SubExt("form-data".into_string()), + vec![(mime::AttrExt("boundary".into_string()), mime::ValueExt(bound.into_string()))] + ) } + /// Guess the MIME type of the `Path` by its extension. /// /// **Guess** is the operative word here, as the contents of a file diff --git a/multipart/src/lib.rs b/multipart/src/lib.rs index 75a11ab43..157ef1620 100644 --- a/multipart/src/lib.rs +++ b/multipart/src/lib.rs @@ -1,17 +1,62 @@ -#![feature(if_let, slicing_syntax, struct_variant, default_type_params)] +#![feature(if_let, slicing_syntax, default_type_params, phase)] extern crate hyper; +#[phase(plugin, link)] extern crate log; + extern crate mime; extern crate serialize; +use self::mime::Mime; + +use std::io::fs::File; +use std::io::{RefReader, IoResult}; + +use server::BoundaryReader; + pub mod client; pub mod server; +pub struct MultipartFile<'a> { + filename: Option, + reader: FileEntryReader<'a>, + content_type: Mime, +} + +pub enum MultipartField<'a> { + TextField(&'a str), + FileField(MultipartFile<'a>), + MultiFiles(Vec>), +} + +pub enum FileEntryReader<'a> { + FileStream(&'a mut File), + OctetStream(&'a mut BoundaryReader), +} + +impl<'a> Reader for FileEntryReader<'a> { + fn read(&mut self, buf: &mut [u8]) -> IoResult{ + match *self { + FileStream(ref mut rdr) => rdr.read(buf), + OctetStream(ref mut rdr) => rdr.read(buf), + } + } +} + #[cfg(test)] mod test { - + use hyper::Url; + use hyper::client::request::Request as ClientReq; + use client::Multipart as ClientMulti; + #[test] - fn client_api_test() { - + fn client_api_test() { + let request = ClientReq::get(Url::parse("http://localhost:1337/").unwrap()).unwrap(); + + let mut multipart = ClientMulti::new(); + + multipart.add_text("hello", "world"); + multipart.add_text("goodnight", "sun"); + + multipart.send(request); } } diff --git a/multipart/src/server.rs b/multipart/src/server.rs index e69de29bb..5cb6f9bc1 100644 --- a/multipart/src/server.rs +++ b/multipart/src/server.rs @@ -0,0 +1,120 @@ +use hyper::header::common::content_type::ContentType; +use hyper::server::request::Request; + +use mime::{Mime, AttrExt, ValueExt}; + +use super::MultipartField; + +use std::io::{RefReader, BufferedReader, IoResult, EndOfFile, standard_error}; + +fn is_multipart_formdata(req: &Request) -> bool { + use mime::{Multipart, Mime, SubExt}; + + req.headers.get::().map(|ct| { + let ContentType(ref mime) = *ct; + match *mime { + Mime(Multipart, SubExt(ref subtype), _) => subtype[] == "form-data", + _ => false, + } + }).unwrap_or(false) +} + +fn get_boundary(ct: &ContentType) -> Option { + let ContentType(ref mime) = *ct; + let Mime(_, _, ref params) = *mime; + + params.iter().find(|&&(ref name, _)| + if let AttrExt(ref name) = *name { + name[] == "boundary" + } else { false } + ).and_then(|&(_, ref val)| + if let ValueExt(ref val) = *val { + Some(val.clone()) + } else { None } + ) +} + +pub struct Multipart { + source: BoundaryReader, +} + +impl Multipart { + + /// If the given `Request` is of `Content-Type: multipart/form-data`, return + /// the wrapped request as `Ok(Multipart)`, otherwise `Err(Request)`. + pub fn from_request(req: Request) -> Result { + if !is_multipart_formdata(&req) { return Err(req); } + let boundary = try!(req.headers.get::().and_then(get_boundary).ok_or_else(|| req)); + + Ok(Multipart { source: BoundaryReader::from_request(req, boundary.into_bytes()) }) + } + + fn read_content_disposition(&mut self) -> ( +} + +impl<'a> Iterator<(String, MultipartField<'a>)> for Multipart { + fn next(&'a mut self) -> Option<(String, MultipartField<'a>)> { + unimplemented!(); + } +} + + +/// A `Reader` that will yield bytes until it sees a given sequence. +pub struct BoundaryReader { + reader: BufferedReader, + boundary: Vec, + last_search_idx: uint, + boundary_read: bool, +} + +impl BoundaryReader { + fn from_request(request: Request, boundary: String) -> BoundaryReader { + BoundaryReader { + reader: BufferedReader::new(request), + boundary: boundary.into_bytes(), + last_search_idx: 0, + boundary_read: false, + } + } + + fn read_to_boundary(&mut self) -> IoResult { + if !self.boundary_read { + let lookahead = try!(self.reader.fill_buf()); + + self.last_search_idx = lookahead[self.last_search_idx..] + .position_elem(&b'-').unwrap_or(lookahead.len() - 1); + + Ok(lookahead[self.last_search_idx..].starts_with(self.boundary[])) + } else if self.last_search_idx == 0 { + Err(standard_error(EndOfFile)) + } else { Ok(true) } + } + + fn consume_boundary(&mut self) { + self.reader.consume(self.boundary.len()); + self.last_search_idx = 0; + self.boundary_read = false; + } + + fn set_boundary(&'a mut self, boundary: String) { + self.boundary = boundary.into_bytes(); + } +} + +impl Reader for BoundaryReader { + fn read(&mut self, buf: &mut [u8]) -> IoResult { + use std::cmp; + + self.boundary_read = try!(self.read_to_boundary()); + + let trunc_len = cmp::min(buf.len(), self.last_search_idx); + + let trunc_buf = buf[mut ..trunc_len]; // Truncate the buffer so we don't read ahead + + let bytes_read = self.reader.read(trunc_buf).unwrap(); + self.last_search_idx -= bytes_read; + + Ok(bytes_read) + } +} + From 0e4e392c9069a27443d0c1dc97cbe2951ccd912c Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Wed, 5 Nov 2014 23:51:50 -0800 Subject: [PATCH 004/453] Implement content disposition parsing --- multipart/src/server.rs | 65 +++++++++++++++++++++++++++++++++++++++-- 1 file changed, 63 insertions(+), 2 deletions(-) diff --git a/multipart/src/server.rs b/multipart/src/server.rs index 5cb6f9bc1..39a85ba5c 100644 --- a/multipart/src/server.rs +++ b/multipart/src/server.rs @@ -5,7 +5,7 @@ use mime::{Mime, AttrExt, ValueExt}; use super::MultipartField; -use std::io::{RefReader, BufferedReader, IoResult, EndOfFile, standard_error}; +use std::io::{RefReader, BufferedReader, IoResult, EndOfFile, standard_error, OtherIoError}; fn is_multipart_formdata(req: &Request) -> bool { use mime::{Multipart, Mime, SubExt}; @@ -49,7 +49,68 @@ impl Multipart { Ok(Multipart { source: BoundaryReader::from_request(req, boundary.into_bytes()) }) } - fn read_content_disposition(&mut self) -> ( + fn read_content_disposition(&mut self) -> IoResult<(String, String, Option)> { + let line = try!(self.source.read_line()); + + // Find the end of CONT_DISP in the line + let disp_type = { + const CONT_DISP: &'static str = "Content-Disposition:"; + + let disp_idx = try_find!(line[], find_str, CONT_DISP, + "Content-Disposition subheader not found!", line) + CONT_DISP.len(); + + let disp_type_end = try_find!(line[disp_idx..], find, ';', + "Error parsing Content-Disposition value!", line); + + line[disp_idx .. disp_type_end].trim().into_string(); + }; + + let field_name = { + const NAME: &'static str = "name=\""; + + let name_idx = try_find!(line[], find_str, NAME, + "Error parsing field name!", line) + NAME.len(); + + let name_end = try_find!(line[name_idx ..], find, '"', + "Error parsing field name!", line); + + line[name_idx .. name_end].into_string(); // No trim here since it's in quotes. + }; + + let filename = { + const FILENAME: &'static str = "filename=\""; + + let filename_idx = line[].find_str(FILENAME).map(|idx| idx + FILENAME.len()); + let filename_idxs = with(filename_idx, |&start| line[start ..].find('"')); + + filename_idxs.map(|(start, end)| line[start .. end].into_string()) + }; + + (disp_type, field_name, filename) + } +} + +fn with(left: Option, right: |&T| -> Option) -> Option<(T, U)> { + let temp = left.as_ref().and_then(right); + match (left, temp) { + Some(lval), Some(rval) => Some((lval, rval)), + _ => None, + } +} + +macro_rules! try_find( + ($haystack:expr, $fn:ident, $needle:expr, $err:expr, $line:expr) => ( + try!($haystack.$fn($needle).ok_or(line_error($err, $line.clone()))) + ) + +fn try_find<'a>(substr: &'a str, needle: &str, err_msg: &'static str + +fn line_error(msg: &'static str, line: String) -> IoError { + IoError { + kind: OtherIoError, + desc: msg, + detail: Some(line), + } } impl<'a> Iterator<(String, MultipartField<'a>)> for Multipart { From a043b606d457ec648114946a316bdcd61dd21006 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sun, 9 Nov 2014 02:34:42 -0800 Subject: [PATCH 005/453] Fix compiler errors --- multipart/src/client.rs | 25 +++++++++++----------- multipart/src/lib.rs | 22 +++++++++---------- multipart/src/multipart_server.rs | 22 +++++++++++++++++++ multipart/src/server.rs | 35 +++++++++++++++++-------------- 4 files changed, 64 insertions(+), 40 deletions(-) create mode 100644 multipart/src/multipart_server.rs diff --git a/multipart/src/client.rs b/multipart/src/client.rs index 0a426328c..75b73a390 100644 --- a/multipart/src/client.rs +++ b/multipart/src/client.rs @@ -19,33 +19,33 @@ use super::{MultipartField, TextField, FileField, FileStream, MultipartFile}; const BOUNDARY_LEN: uint = 8; -pub struct Multipart<'a> { - fields: Vec<(&'a str, MultipartField<'a>)>, +pub struct Multipart { + fields: Vec<(String, MultipartField)>, boundary: String, } /// Shorthand for a writable request (`Request`) type ReqWrite = Request; -impl<'a> Multipart<'a> { +impl Multipart { - pub fn new() -> Multipart<'a> { + pub fn new() -> Multipart { Multipart { fields: Vec::new(), boundary: random_alphanumeric(BOUNDARY_LEN), } } - pub fn add_text(&mut self, name: &'a str, val: &'a str) { - self.fields.push((name, TextField(val))); + pub fn add_text(&mut self, name: &str, val: &str) { + self.fields.push((name.into_string(), TextField(val.into_string()))); } /// Add the file to the multipart request, guessing its `Content-Type` from its extension - pub fn add_file(&mut self, name: &'a str, file: &'a mut File) { + pub fn add_file(&mut self, name: &str, file: File) { let filename = file.path().filename_str().map(|s| s.into_string()); let content_type = guess_mime_type(file.path()); - self.fields.push((name, FileField(MultipartFile { + self.fields.push((name.into_string(), FileField(MultipartFile { filename: filename, reader: FileStream(file), content_type: content_type, @@ -79,9 +79,8 @@ impl<'a> Multipart<'a> { try!(match field { TextField(text) => req.write(b"\r\n\r\n") - .and_then(|_| write_line(req, text)), // Style suggestions welcome + .and_then(|_| write_line(req, &*text)), // Style suggestions welcome FileField(file) => write_file(req, file), - _ => unimplemented!(), }); try!(write_boundary(req, boundary[])); @@ -96,7 +95,7 @@ fn write_boundary(req: &mut ReqWrite, boundary: &str) -> IoResult<()> { write!(req, "--{}\r\n", boundary) } -fn write_file<'a>(req: &mut ReqWrite, mut file: MultipartFile<'a>) -> IoResult<()> { +fn write_file(req: &mut ReqWrite, mut file: MultipartFile) -> IoResult<()> { try!(file.filename.map(|filename| write!(req, "; filename=\"{}\"\r\n", filename)).unwrap_or(Ok(()))); try!(write!(req, "Content-Type: {}\r\n\r\n", file.content_type)); io::util::copy(&mut file.reader, req) @@ -155,7 +154,7 @@ pub fn get_mime_type(ext: &str) -> Mime { return mime_type.clone(); } - let mime_type = find_mime_type(&ext); + let mime_type = find_mime_type(&*ext); cache.borrow_mut().insert(ext, mime_type.clone()); @@ -165,7 +164,7 @@ pub fn get_mime_type(ext: &str) -> Mime { const MIME_TYPES: &'static str = include_str!("../mime_types.json"); /// Load the MIME_TYPES as JSON and try to locate `ext` -fn find_mime_type(ext: &String) -> Mime { +fn find_mime_type(ext: &str) -> Mime { json::from_str(MIME_TYPES).unwrap() .find(ext).and_then(|j| j.as_string()) .and_then(from_str::) diff --git a/multipart/src/lib.rs b/multipart/src/lib.rs index 157ef1620..c05dee8bd 100644 --- a/multipart/src/lib.rs +++ b/multipart/src/lib.rs @@ -1,4 +1,4 @@ -#![feature(if_let, slicing_syntax, default_type_params, phase)] +#![feature(if_let, slicing_syntax, default_type_params, phase, macro_rules)] extern crate hyper; #[phase(plugin, link)] extern crate log; @@ -15,24 +15,24 @@ use server::BoundaryReader; pub mod client; pub mod server; -pub struct MultipartFile<'a> { +pub struct MultipartFile { filename: Option, - reader: FileEntryReader<'a>, + reader: FileEntryReader, content_type: Mime, } -pub enum MultipartField<'a> { - TextField(&'a str), - FileField(MultipartFile<'a>), - MultiFiles(Vec>), +pub enum MultipartField { + TextField(String), + FileField(MultipartFile), + // MultiFiles(Vec), /* TODO: Multiple files */ } -pub enum FileEntryReader<'a> { - FileStream(&'a mut File), - OctetStream(&'a mut BoundaryReader), +pub enum FileEntryReader { + FileStream(File), + OctetStream(BoundaryReader), } -impl<'a> Reader for FileEntryReader<'a> { +impl Reader for FileEntryReader { fn read(&mut self, buf: &mut [u8]) -> IoResult{ match *self { FileStream(ref mut rdr) => rdr.read(buf), diff --git a/multipart/src/multipart_server.rs b/multipart/src/multipart_server.rs new file mode 100644 index 000000000..bde4b240c --- /dev/null +++ b/multipart/src/multipart_server.rs @@ -0,0 +1,22 @@ +extern crate hyper; + +use self::hyper::server::{Server, Incoming}; +use self::hyper::status; + +use std::io::net::ip::Ipv4Addr; + +fn hello(mut incoming: Incoming) { + for (mut req, mut res) in incoming { + *res.status_mut() = status::Ok; + + println!("{}", req.headers); + println!("{}", req.read_to_string().unwrap()); + + res.start().unwrap().end().unwrap(); + } +} + +fn main() { + let server = Server::http(Ipv4Addr(127, 0, 0, 1), 1337); + server.listen(hello).unwrap(); +} diff --git a/multipart/src/server.rs b/multipart/src/server.rs index 39a85ba5c..8432cc1d2 100644 --- a/multipart/src/server.rs +++ b/multipart/src/server.rs @@ -5,7 +5,7 @@ use mime::{Mime, AttrExt, ValueExt}; use super::MultipartField; -use std::io::{RefReader, BufferedReader, IoResult, EndOfFile, standard_error, OtherIoError}; +use std::io::{RefReader, BufferedReader, IoError, IoResult, EndOfFile, standard_error, OtherIoError}; fn is_multipart_formdata(req: &Request) -> bool { use mime::{Multipart, Mime, SubExt}; @@ -38,19 +38,27 @@ pub struct Multipart { source: BoundaryReader, } +macro_rules! try_find( + ($haystack:expr, $f:ident, $needle:expr, $err:expr, $line:expr) => ( + try!($haystack.$f($needle).ok_or(line_error($err, $line.clone()))) + ) +) + impl Multipart { /// If the given `Request` is of `Content-Type: multipart/form-data`, return /// the wrapped request as `Ok(Multipart)`, otherwise `Err(Request)`. pub fn from_request(req: Request) -> Result { if !is_multipart_formdata(&req) { return Err(req); } - let boundary = try!(req.headers.get::().and_then(get_boundary).ok_or_else(|| req)); - Ok(Multipart { source: BoundaryReader::from_request(req, boundary.into_bytes()) }) + let boundary = if let Some(boundary) = req.headers.get::() + .and_then(get_boundary) { boundary } else { return Err(req); }; + + Ok(Multipart { source: BoundaryReader::from_request(req, boundary) }) } fn read_content_disposition(&mut self) -> IoResult<(String, String, Option)> { - let line = try!(self.source.read_line()); + let line = try!(self.source.reader.read_line()); // Find the end of CONT_DISP in the line let disp_type = { @@ -62,7 +70,7 @@ impl Multipart { let disp_type_end = try_find!(line[disp_idx..], find, ';', "Error parsing Content-Disposition value!", line); - line[disp_idx .. disp_type_end].trim().into_string(); + line[disp_idx .. disp_type_end].trim().into_string() }; let field_name = { @@ -74,7 +82,7 @@ impl Multipart { let name_end = try_find!(line[name_idx ..], find, '"', "Error parsing field name!", line); - line[name_idx .. name_end].into_string(); // No trim here since it's in quotes. + line[name_idx .. name_end].into_string() // No trim here since it's in quotes. }; let filename = { @@ -86,24 +94,19 @@ impl Multipart { filename_idxs.map(|(start, end)| line[start .. end].into_string()) }; - (disp_type, field_name, filename) + Ok((disp_type, field_name, filename)) } } fn with(left: Option, right: |&T| -> Option) -> Option<(T, U)> { let temp = left.as_ref().and_then(right); match (left, temp) { - Some(lval), Some(rval) => Some((lval, rval)), + (Some(lval), Some(rval)) => Some((lval, rval)), _ => None, } } -macro_rules! try_find( - ($haystack:expr, $fn:ident, $needle:expr, $err:expr, $line:expr) => ( - try!($haystack.$fn($needle).ok_or(line_error($err, $line.clone()))) - ) -fn try_find<'a>(substr: &'a str, needle: &str, err_msg: &'static str fn line_error(msg: &'static str, line: String) -> IoError { IoError { @@ -113,8 +116,8 @@ fn line_error(msg: &'static str, line: String) -> IoError { } } -impl<'a> Iterator<(String, MultipartField<'a>)> for Multipart { - fn next(&'a mut self) -> Option<(String, MultipartField<'a>)> { +impl Iterator<(String, MultipartField)> for Multipart { + fn next(&mut self) -> Option<(String, MultipartField)> { unimplemented!(); } } @@ -157,7 +160,7 @@ impl BoundaryReader { self.boundary_read = false; } - fn set_boundary(&'a mut self, boundary: String) { + fn set_boundary(&mut self, boundary: String) { self.boundary = boundary.into_bytes(); } } From 8cbe681950ab43f8ce27e4855feb872df2e976da Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Mon, 10 Nov 2014 01:52:24 -0800 Subject: [PATCH 006/453] More work, compiles successfully --- multipart/src/client.rs | 81 +++----------------------- multipart/src/lib.rs | 119 ++++++++++++++++++++++++++++++++----- multipart/src/server.rs | 126 ++++++++++++++++++++++++++++++++++++---- 3 files changed, 230 insertions(+), 96 deletions(-) diff --git a/multipart/src/client.rs b/multipart/src/client.rs index 75b73a390..26bc98069 100644 --- a/multipart/src/client.rs +++ b/multipart/src/client.rs @@ -5,31 +5,27 @@ use hyper::{HttpResult, HttpIoError}; use mime::{mod, Mime}; -use serialize::json; - -use std::cell::RefCell; - -use std::collections::HashMap; +use mime_guess::guess_mime_type; use std::io::IoResult; use std::io::fs::File; use std::io; -use super::{MultipartField, TextField, FileField, FileStream, MultipartFile}; +use super::{MultipartField, TextField, FileField, MultipartFile}; const BOUNDARY_LEN: uint = 8; -pub struct Multipart { - fields: Vec<(String, MultipartField)>, +pub struct Multipart<'a> { + fields: Vec<(String, MultipartField<'a>)>, boundary: String, } /// Shorthand for a writable request (`Request`) type ReqWrite = Request; -impl Multipart { +impl<'a> Multipart<'a> { - pub fn new() -> Multipart { + pub fn new() -> Multipart<'a> { Multipart { fields: Vec::new(), boundary: random_alphanumeric(BOUNDARY_LEN), @@ -41,15 +37,12 @@ impl Multipart { } /// Add the file to the multipart request, guessing its `Content-Type` from its extension - pub fn add_file(&mut self, name: &str, file: File) { + pub fn add_file(&mut self, name: &str, file: &'a mut File) { let filename = file.path().filename_str().map(|s| s.into_string()); let content_type = guess_mime_type(file.path()); - self.fields.push((name.into_string(), FileField(MultipartFile { - filename: filename, - reader: FileStream(file), - content_type: content_type, - }))) + self.fields.push((name.into_string(), + FileField(MultipartFile::from_file(filename, file, content_type)))); } /// Apply the appropriate headers to the `Request` and send the data. @@ -125,60 +118,4 @@ fn multipart_mime(bound: &str) -> Mime { } -/// Guess the MIME type of the `Path` by its extension. -/// -/// **Guess** is the operative word here, as the contents of a file -/// may not or may not match its MIME type/extension. -pub fn guess_mime_type(path: &Path) -> Mime { - let ext = path.extension_str().unwrap_or(""); - - get_mime_type(ext) -} - -local_data_key!(mime_types_cache: RefCell>) - -/// Get the MIME type associated with a file extension -// MIME Types are cached in a task-local heap -pub fn get_mime_type(ext: &str) -> Mime { - if ext.is_empty() { return octet_stream(); } - - let ext = ext.into_string(); - - let cache = if let Some(cache) = mime_types_cache.get() { cache } - else { - mime_types_cache.replace(Some(RefCell::new(HashMap::new()))); - mime_types_cache.get().unwrap() - }; - - if let Some(mime_type) = cache.borrow().find(&ext) { - return mime_type.clone(); - } - - let mime_type = find_mime_type(&*ext); - - cache.borrow_mut().insert(ext, mime_type.clone()); - - mime_type -} - -const MIME_TYPES: &'static str = include_str!("../mime_types.json"); - -/// Load the MIME_TYPES as JSON and try to locate `ext` -fn find_mime_type(ext: &str) -> Mime { - json::from_str(MIME_TYPES).unwrap() - .find(ext).and_then(|j| j.as_string()) - .and_then(from_str::) - .unwrap_or_else(octet_stream) -} - -fn octet_stream() -> Mime { - Mime(mime::Application, mime::SubExt("octet-stream".into_string()), Vec::new()) -} - -#[test] -fn test_mime_type_guessing() { - assert!(get_mime_type("gif").to_string() == "image/gif".to_string()); - assert!(get_mime_type("txt").to_string() == "text/plain".to_string()); - assert!(get_mime_type("blahblah").to_string() == "application/octet-stream".to_string()); -} diff --git a/multipart/src/lib.rs b/multipart/src/lib.rs index c05dee8bd..9c18b09f8 100644 --- a/multipart/src/lib.rs +++ b/multipart/src/lib.rs @@ -7,38 +7,129 @@ extern crate serialize; use self::mime::Mime; +use std::kinds::marker; use std::io::fs::File; -use std::io::{RefReader, IoResult}; +use std::io::{AsRefReader, RefReader, IoResult}; use server::BoundaryReader; pub mod client; pub mod server; -pub struct MultipartFile { +pub struct MultipartFile<'a> { + _marker: marker::ContravariantLifetime<'a>, filename: Option, - reader: FileEntryReader, - content_type: Mime, + content_type: Mime, + reader: &'a mut Reader +'a , } -pub enum MultipartField { +impl<'a> MultipartFile<'a> { + fn from_octet(filename: Option, reader: &'a mut Reader, cont_type: &str) -> MultipartFile<'a> { + MultipartFile { + filename: filename, + reader: reader, + content_type: from_str(cont_type).unwrap_or_else(mime_guess::octet_stream), + _marker: marker::ContravariantLifetime, + } + } + + fn from_file(filename: Option, reader: &'a mut File, mime: Mime) -> MultipartFile<'a> { + MultipartFile { + filename: filename, + reader: reader, + content_type: mime, + _marker: marker::ContravariantLifetime, + } + } +} + + +pub enum MultipartField<'a> { TextField(String), - FileField(MultipartFile), + FileField(MultipartFile<'a>), // MultiFiles(Vec), /* TODO: Multiple files */ } -pub enum FileEntryReader { - FileStream(File), - OctetStream(BoundaryReader), +impl<'a> Reader for MultipartFile<'a> { + fn read(&mut self, buf: &mut [u8]) -> IoResult{ + self.reader.read(buf) + } } -impl Reader for FileEntryReader { - fn read(&mut self, buf: &mut [u8]) -> IoResult{ - match *self { - FileStream(ref mut rdr) => rdr.read(buf), - OctetStream(ref mut rdr) => rdr.read(buf), +pub mod mime_guess { + + use mime::{mod, Mime}; + + use serialize::json; + + use std::cell::RefCell; + + use std::collections::HashMap; + + + /// Guess the MIME type of the `Path` by its extension. + /// + /// **Guess** is the operative word here, as the contents of a file + /// may not or may not match its MIME type/extension. + pub fn guess_mime_type(path: &Path) -> Mime { + let ext = path.extension_str().unwrap_or(""); + + get_mime_type(ext) + } + + pub fn guess_mime_type_filename(filename: &str) -> Mime { + let path = Path::new(filename); + + guess_mime_type(&path) + } + + local_data_key!(mime_types_cache: RefCell>) + + /// Get the MIME type associated with a file extension + // MIME Types are cached in a task-local heap + pub fn get_mime_type(ext: &str) -> Mime { + if ext.is_empty() { return octet_stream(); } + + let ext = ext.into_string(); + + let cache = if let Some(cache) = mime_types_cache.get() { cache } + else { + mime_types_cache.replace(Some(RefCell::new(HashMap::new()))); + mime_types_cache.get().unwrap() + }; + + if let Some(mime_type) = cache.borrow().find(&ext) { + return mime_type.clone(); } + + let mime_type = find_mime_type(&*ext); + + cache.borrow_mut().insert(ext, mime_type.clone()); + + mime_type + } + + const MIME_TYPES: &'static str = include_str!("../mime_types.json"); + + /// Load the MIME_TYPES as JSON and try to locate `ext` + fn find_mime_type(ext: &str) -> Mime { + json::from_str(MIME_TYPES).unwrap() + .find(ext).and_then(|j| j.as_string()) + .and_then(from_str::) + .unwrap_or_else(octet_stream) + } + + pub fn octet_stream() -> Mime { + Mime(mime::Application, mime::SubExt("octet-stream".into_string()), Vec::new()) + } + +#[test] + fn test_mime_type_guessing() { + assert!(get_mime_type("gif").to_string() == "image/gif".to_string()); + assert!(get_mime_type("txt").to_string() == "text/plain".to_string()); + assert!(get_mime_type("blahblah").to_string() == "application/octet-stream".to_string()); } + } #[cfg(test)] diff --git a/multipart/src/server.rs b/multipart/src/server.rs index 8432cc1d2..4b1194c71 100644 --- a/multipart/src/server.rs +++ b/multipart/src/server.rs @@ -3,10 +3,12 @@ use hyper::server::request::Request; use mime::{Mime, AttrExt, ValueExt}; -use super::MultipartField; +use super::{MultipartField, TextField, MultipartFile, FileField}; use std::io::{RefReader, BufferedReader, IoError, IoResult, EndOfFile, standard_error, OtherIoError}; +use std::kinds::marker; + fn is_multipart_formdata(req: &Request) -> bool { use mime::{Multipart, Mime, SubExt}; @@ -56,6 +58,46 @@ impl Multipart { Ok(Multipart { source: BoundaryReader::from_request(req, boundary) }) } + + pub fn read_entry<'a>(&'a mut self) -> IoResult<(String, MultipartField<'a>)> { + let (disp_type, field_name, filename) = try!(self.read_content_disposition()); + + if &*disp_type != "form-data" { + return Err(IoError { + kind: OtherIoError, + desc: "Content-Disposition value was not \"form-data\"", + detail: Some(format!("Content-Disposition: {}", disp_type)), + }); + } + + if let Some(content_type) = try!(self.read_content_type()) { + let _ = try!(self.source.reader.read_line()); // Consume empty line + Ok((field_name, FileField(MultipartFile::from_octet(filename, &mut self.source, content_type[])))) + } else { + // Empty line consumed by read_content_type() + let text = try!(self.source.read_to_string()); + Ok((field_name, TextField(text))) + } + } + + /// Call `f` for each entry in the multipart request. + /// This is a substitute for `Multipart` implementing `Iterator`, + /// since `Iterator::next()` can't use bound lifetimes. + /// See https://www.reddit.com/r/rust/comments/2lkk4i/concrete_lifetime_vs_bound_lifetime/ + pub fn foreach_entry(&mut self, f: <'a>|String, MultipartField<'a>|) { + loop { + match self.read_entry() { + Ok((name, field)) => f(name, field), + Err(err) => { + if err.kind != EndOfFile { + error!("Error reading Multipart: {}", err); + } + + break; + }, + } + } + } fn read_content_disposition(&mut self) -> IoResult<(String, String, Option)> { let line = try!(self.source.reader.read_line()); @@ -96,6 +138,18 @@ impl Multipart { Ok((disp_type, field_name, filename)) } + + fn read_content_type(&mut self) -> IoResult> { + let line = try!(self.source.reader.read_line()); + + const CONTENT_TYPE: &'static str = "Content-Type:"; + + let type_idx = (&*line).find_str(CONTENT_TYPE); + + // FIXME Will not properly parse for multiple files! + // Does not expect boundary= + Ok(type_idx.map(|start| line[start + CONTENT_TYPE.len()..].trim().into_string())) + } } fn with(left: Option, right: |&T| -> Option) -> Option<(T, U)> { @@ -106,8 +160,6 @@ fn with(left: Option, right: |&T| -> Option) -> Option<(T, U)> { } } - - fn line_error(msg: &'static str, line: String) -> IoError { IoError { kind: OtherIoError, @@ -116,12 +168,22 @@ fn line_error(msg: &'static str, line: String) -> IoError { } } -impl Iterator<(String, MultipartField)> for Multipart { - fn next(&mut self) -> Option<(String, MultipartField)> { - unimplemented!(); +/* FIXME: Can't have an iterator return a borrowed reference +impl<'a> Iterator<(String, MultipartField<'a>)> for Multipart { + fn next(&mut self) -> Option<(String, MultipartField<'a>)> { + match self.read_entry() { + Ok(ok) => Some(ok), + Err(err) => { + if err.kind != EndOfFile { + error!("Error reading Multipart: {}", err); + } + + None + }, + } } } - +*/ /// A `Reader` that will yield bytes until it sees a given sequence. pub struct BoundaryReader { @@ -132,7 +194,9 @@ pub struct BoundaryReader { } impl BoundaryReader { - fn from_request(request: Request, boundary: String) -> BoundaryReader { + fn from_request(request: Request, mut boundary: String) -> BoundaryReader { + boundary.prepend("--"); + BoundaryReader { reader: BufferedReader::new(request), boundary: boundary.into_bytes(), @@ -155,7 +219,7 @@ impl BoundaryReader { } fn consume_boundary(&mut self) { - self.reader.consume(self.boundary.len()); + self.reader.consume(self.last_search_idx + self.boundary.len()); self.last_search_idx = 0; self.boundary_read = false; } @@ -179,6 +243,48 @@ impl Reader for BoundaryReader { self.last_search_idx -= bytes_read; Ok(bytes_read) - } + } +} + +trait Prepend { + fn prepend(&mut self, t: T); +} + +impl Prepend for String { + fn prepend(&mut self, s: S) { + unsafe { + self.as_mut_vec().prepend(s.as_slice().as_bytes()); + } + } +} + +impl<'a, T> Prepend<&'a [T]> for Vec { + fn prepend(&mut self, slice: &[T]) { + use std::ptr::copy_memory; + + let old_len = self.len(); + + self.reserve(slice.len()); + + unsafe { + self.set_len(old_len + slice.len()); + copy_memory(self[mut slice.len()..].as_mut_ptr(), self[..old_len].as_ptr(), old_len); + copy_memory(self.as_mut_ptr(), slice.as_ptr(), slice.len()); + } + } +} + +#[test] +fn test_prepend() { + let mut vec = vec![3u64, 4, 5]; + vec.prepend([1u64, 2]); + assert_eq!(vec[], [1u64, 2, 3, 4, 5][]); +} + +#[test] +fn test_prepend_string() { + let mut string = "World!".into_string(); + string.prepend("Hello, "); + assert_eq!(&*string, "Hello, World!"); } From f572f16a187a379f04e160742225bceb974f01e6 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Mon, 10 Nov 2014 15:50:46 -0800 Subject: [PATCH 007/453] Build multipart_server correctly --- multipart/Cargo.toml | 2 ++ multipart/src/{ => bin}/multipart_server.rs | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) rename multipart/src/{ => bin}/multipart_server.rs (81%) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 111eb7a31..054cd172a 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -6,3 +6,5 @@ authors = ["Austin Bonander "] [dependencies.hyper] git = "https://github.com/hyperium/hyper" + + diff --git a/multipart/src/multipart_server.rs b/multipart/src/bin/multipart_server.rs similarity index 81% rename from multipart/src/multipart_server.rs rename to multipart/src/bin/multipart_server.rs index bde4b240c..67bcf5cee 100644 --- a/multipart/src/multipart_server.rs +++ b/multipart/src/bin/multipart_server.rs @@ -6,7 +6,9 @@ use self::hyper::status; use std::io::net::ip::Ipv4Addr; fn hello(mut incoming: Incoming) { - for (mut req, mut res) in incoming { + for connection in incoming { + let (mut req, mut res) = connection.open().unwrap(); + *res.status_mut() = status::Ok; println!("{}", req.headers); From fad9fc4ff5df34df7b8266e2aa3f8519fc464dd0 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Mon, 10 Nov 2014 15:52:32 -0800 Subject: [PATCH 008/453] Unboxed closure ICE --- multipart/src/server.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/src/server.rs b/multipart/src/server.rs index 4b1194c71..e7eb5d011 100644 --- a/multipart/src/server.rs +++ b/multipart/src/server.rs @@ -84,7 +84,7 @@ impl Multipart { /// This is a substitute for `Multipart` implementing `Iterator`, /// since `Iterator::next()` can't use bound lifetimes. /// See https://www.reddit.com/r/rust/comments/2lkk4i/concrete_lifetime_vs_bound_lifetime/ - pub fn foreach_entry(&mut self, f: <'a>|String, MultipartField<'a>|) { + pub fn foreach_entry FnMut(String, MultipartField<'a>)>(&mut self, f: F) { loop { match self.read_entry() { Ok((name, field)) => f(name, field), From 9b28fd900cc51e9c7deb6441ce71997488ee652f Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Mon, 10 Nov 2014 16:02:38 -0800 Subject: [PATCH 009/453] updated lifetime syntax --- multipart/src/server.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/src/server.rs b/multipart/src/server.rs index 4b1194c71..589bc25b7 100644 --- a/multipart/src/server.rs +++ b/multipart/src/server.rs @@ -84,7 +84,7 @@ impl Multipart { /// This is a substitute for `Multipart` implementing `Iterator`, /// since `Iterator::next()` can't use bound lifetimes. /// See https://www.reddit.com/r/rust/comments/2lkk4i/concrete_lifetime_vs_bound_lifetime/ - pub fn foreach_entry(&mut self, f: <'a>|String, MultipartField<'a>|) { + pub fn foreach_entry(&mut self, f: for<'a>|String, MultipartField<'a>|) { loop { match self.read_entry() { Ok((name, field)) => f(name, field), From 4a25e72842dd18c7a45950e2ccecec8ae0615374 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Tue, 11 Nov 2014 18:38:49 -0800 Subject: [PATCH 010/453] More work, compiles but doesn't work right --- multipart/src/bin/multipart_server.rs | 13 +- multipart/src/client.rs | 15 +- multipart/src/lib.rs | 12 +- multipart/src/server.rs | 290 ++++++++++++++++++++------ 4 files changed, 252 insertions(+), 78 deletions(-) diff --git a/multipart/src/bin/multipart_server.rs b/multipart/src/bin/multipart_server.rs index 67bcf5cee..17fa556a5 100644 --- a/multipart/src/bin/multipart_server.rs +++ b/multipart/src/bin/multipart_server.rs @@ -1,19 +1,23 @@ extern crate hyper; +extern crate multipart; use self::hyper::server::{Server, Incoming}; use self::hyper::status; +use self::multipart::server::Multipart; + use std::io::net::ip::Ipv4Addr; fn hello(mut incoming: Incoming) { for connection in incoming { - let (mut req, mut res) = connection.open().unwrap(); + let (mut req, mut res) = connection.open().ok().expect("Connection failed!"); + + let mut multipart = Multipart::from_request(req).ok().expect("Could not create multipart!"); + + multipart.foreach_entry(|name, content| println!("Name: {} Content: {}", name, content)); *res.status_mut() = status::Ok; - println!("{}", req.headers); - println!("{}", req.read_to_string().unwrap()); - res.start().unwrap().end().unwrap(); } } @@ -21,4 +25,5 @@ fn hello(mut incoming: Incoming) { fn main() { let server = Server::http(Ipv4Addr(127, 0, 0, 1), 1337); server.listen(hello).unwrap(); + } diff --git a/multipart/src/client.rs b/multipart/src/client.rs index 26bc98069..7e1339077 100644 --- a/multipart/src/client.rs +++ b/multipart/src/client.rs @@ -47,8 +47,13 @@ impl<'a> Multipart<'a> { /// Apply the appropriate headers to the `Request` and send the data. pub fn send(self, mut req: Request) -> HttpResult { + use hyper::method; + assert!(req.method() == method::Post, "Multipart request must use POST method!"); + self.apply_headers(&mut req); + debug!("Fields: {}; Boundary: {}", self.fields[], self.boundary[]); + debug!("{}", req.headers()); let mut req = try!(req.start()); @@ -68,11 +73,10 @@ impl<'a> Multipart<'a> { try!(write_boundary(req, boundary[])); for (name, field) in fields.into_iter() { - try!(write!(req, "Content-Disposition: form-data; name=\"{}\"", name)); + try!(write!(req, "Content-Disposition: form-data; name=\"{}\"\r\n\r\n", name)); try!(match field { - TextField(text) => req.write(b"\r\n\r\n") - .and_then(|_| write_line(req, &*text)), // Style suggestions welcome + TextField(text) => write_line(req, &*text), // Style suggestions welcome FileField(file) => write_file(req, file), }); @@ -102,8 +106,9 @@ fn write_line(req: &mut ReqWrite, s: &str) -> IoResult<()> { /// Generate a random alphanumeric sequence of length `len` fn random_alphanumeric(len: uint) -> String { use std::rand::{task_rng, Rng}; - - task_rng().gen_ascii_chars().take(len).collect() + use std::char::to_lowercase; + + task_rng().gen_ascii_chars().map(to_lowercase).take(len).collect() } fn io_to_http(res: IoResult) -> HttpResult { diff --git a/multipart/src/lib.rs b/multipart/src/lib.rs index 9c18b09f8..ae772d681 100644 --- a/multipart/src/lib.rs +++ b/multipart/src/lib.rs @@ -7,6 +7,7 @@ extern crate serialize; use self::mime::Mime; +use std::fmt::{Formatter, FormatError, Show}; use std::kinds::marker; use std::io::fs::File; use std::io::{AsRefReader, RefReader, IoResult}; @@ -16,6 +17,7 @@ use server::BoundaryReader; pub mod client; pub mod server; + pub struct MultipartFile<'a> { _marker: marker::ContravariantLifetime<'a>, filename: Option, @@ -43,7 +45,13 @@ impl<'a> MultipartFile<'a> { } } +impl<'a> Show for MultipartFile<'a> { + fn fmt(&self, fmt: &mut Formatter) -> Result<(), FormatError> { + write!(fmt, "Filename: {} Content-Type: {}", self.filename, self.content_type) + } +} +#[deriving(Show)] pub enum MultipartField<'a> { TextField(String), FileField(MultipartFile<'a>), @@ -140,14 +148,14 @@ mod test { #[test] fn client_api_test() { - let request = ClientReq::get(Url::parse("http://localhost:1337/").unwrap()).unwrap(); + let request = ClientReq::post(Url::parse("http://localhost:1337/").unwrap()).unwrap(); let mut multipart = ClientMulti::new(); multipart.add_text("hello", "world"); multipart.add_text("goodnight", "sun"); - multipart.send(request); + multipart.send(request).unwrap(); } } diff --git a/multipart/src/server.rs b/multipart/src/server.rs index 589bc25b7..e71ea5918 100644 --- a/multipart/src/server.rs +++ b/multipart/src/server.rs @@ -5,6 +5,8 @@ use mime::{Mime, AttrExt, ValueExt}; use super::{MultipartField, TextField, MultipartFile, FileField}; +use std::cmp; + use std::io::{RefReader, BufferedReader, IoError, IoResult, EndOfFile, standard_error, OtherIoError}; use std::kinds::marker; @@ -37,7 +39,7 @@ fn get_boundary(ct: &ContentType) -> Option { } pub struct Multipart { - source: BoundaryReader, + source: BoundaryReader, } macro_rules! try_find( @@ -50,16 +52,32 @@ impl Multipart { /// If the given `Request` is of `Content-Type: multipart/form-data`, return /// the wrapped request as `Ok(Multipart)`, otherwise `Err(Request)`. - pub fn from_request(req: Request) -> Result { + pub fn from_request(mut req: Request) -> Result { + use std::io::stdio::stdout_raw; + if !is_multipart_formdata(&req) { return Err(req); } let boundary = if let Some(boundary) = req.headers.get::() .and_then(get_boundary) { boundary } else { return Err(req); }; - Ok(Multipart { source: BoundaryReader::from_request(req, boundary) }) + Ok(Multipart { source: BoundaryReader::from_reader(req, format!("--{}\r\n", boundary)) }) } pub fn read_entry<'a>(&'a mut self) -> IoResult<(String, MultipartField<'a>)> { + debug!("Read entry!"); + + loop { + let string = self.source.read_to_string().unwrap(); + self.source.consume_boundary(); + + if string.is_empty() { + break; + } + + debug!("{}", string); + } + + try!(self.source.consume_boundary()); let (disp_type, field_name, filename) = try!(self.read_content_disposition()); if &*disp_type != "form-data" { @@ -71,12 +89,15 @@ impl Multipart { } if let Some(content_type) = try!(self.read_content_type()) { - let _ = try!(self.source.reader.read_line()); // Consume empty line + let _ = try!(self.source.read_line()); // Consume empty line Ok((field_name, FileField(MultipartFile::from_octet(filename, &mut self.source, content_type[])))) } else { // Empty line consumed by read_content_type() let text = try!(self.source.read_to_string()); - Ok((field_name, TextField(text))) + // The last two characters are "\r\n". + // We can't do a simple trim because the content might be terminated + // with line separators we want to preserve. + Ok((field_name, TextField(text[..text.len() - 2].into_string()))) } } @@ -86,6 +107,8 @@ impl Multipart { /// See https://www.reddit.com/r/rust/comments/2lkk4i/concrete_lifetime_vs_bound_lifetime/ pub fn foreach_entry(&mut self, f: for<'a>|String, MultipartField<'a>|) { loop { + debug!("Loop!"); + match self.read_entry() { Ok((name, field)) => f(name, field), Err(err) => { @@ -100,20 +123,29 @@ impl Multipart { } fn read_content_disposition(&mut self) -> IoResult<(String, String, Option)> { - let line = try!(self.source.reader.read_line()); + debug!("Read content disposition!"); + let line = try!(self.source.read_line()); + + debug!("Line: {}", line); // Find the end of CONT_DISP in the line let disp_type = { const CONT_DISP: &'static str = "Content-Disposition:"; let disp_idx = try_find!(line[], find_str, CONT_DISP, - "Content-Disposition subheader not found!", line) + CONT_DISP.len(); + "Content-Disposition subheader not found!", line) + CONT_DISP.len(); + + debug!("Disp idx: {} Line len: {}", disp_idx, line.len()); let disp_type_end = try_find!(line[disp_idx..], find, ';', "Error parsing Content-Disposition value!", line); - line[disp_idx .. disp_type_end].trim().into_string() + debug!("Disp end: {}", disp_type_end); + + line[disp_idx .. disp_idx + disp_type_end].trim().into_string() }; + + debug!("Disp-type: {}", disp_type); let field_name = { const NAME: &'static str = "name=\""; @@ -121,26 +153,35 @@ impl Multipart { let name_idx = try_find!(line[], find_str, NAME, "Error parsing field name!", line) + NAME.len(); + debug!("Name idx: {}", name_idx); + let name_end = try_find!(line[name_idx ..], find, '"', "Error parsing field name!", line); - line[name_idx .. name_end].into_string() // No trim here since it's in quotes. + debug!("Name end: {}", name_end); + + line[name_idx .. name_idx + name_end].into_string() // No trim here since it's in quotes. }; + debug!("Field name: {}", field_name); + let filename = { const FILENAME: &'static str = "filename=\""; let filename_idx = line[].find_str(FILENAME).map(|idx| idx + FILENAME.len()); let filename_idxs = with(filename_idx, |&start| line[start ..].find('"')); - filename_idxs.map(|(start, end)| line[start .. end].into_string()) + filename_idxs.map(|(start, end)| line[start .. start + end].into_string()) }; + + debug!("Filename: {}", filename); Ok((disp_type, field_name, filename)) } fn read_content_type(&mut self) -> IoResult> { - let line = try!(self.source.reader.read_line()); + debug!("Read content type!"); + let line = try!(self.source.read_line()); const CONTENT_TYPE: &'static str = "Content-Type:"; @@ -186,105 +227,220 @@ impl<'a> Iterator<(String, MultipartField<'a>)> for Multipart { */ /// A `Reader` that will yield bytes until it sees a given sequence. -pub struct BoundaryReader { - reader: BufferedReader, +pub struct BoundaryReader { + reader: S, boundary: Vec, last_search_idx: uint, boundary_read: bool, + buf: Vec, + buf_len: uint, +} + +fn eof() -> IoResult { + Err(standard_error(EndOfFile)) } -impl BoundaryReader { - fn from_request(request: Request, mut boundary: String) -> BoundaryReader { - boundary.prepend("--"); +const BUF_SIZE: uint = 1024 * 64; // 64k buffer + +impl BoundaryReader where S: Reader { + fn from_reader(reader: S, boundary: String) -> BoundaryReader { + let mut buf = Vec::with_capacity(BUF_SIZE); + unsafe { buf.set_len(BUF_SIZE); } BoundaryReader { - reader: BufferedReader::new(request), + reader: reader, boundary: boundary.into_bytes(), last_search_idx: 0, - boundary_read: false, + boundary_read: false, + buf: buf, + buf_len: 0, } } - fn read_to_boundary(&mut self) -> IoResult { + fn read_to_boundary(&mut self) -> IoResult<()> { + debug!("Read to boundary!"); + if !self.boundary_read { - let lookahead = try!(self.reader.fill_buf()); - - self.last_search_idx = lookahead[self.last_search_idx..] - .position_elem(&b'-').unwrap_or(lookahead.len() - 1); + try!(self.true_fill_buf()); + + debug!("Exited true_fill_buf"); + + if self.buf_len == 0 { return eof(); } + + let lookahead = self.buf[self.last_search_idx .. self.buf_len]; + + debug!("Buf len: {}", self.buf_len); + + debug!("Lookahead: {}", lookahead.to_ascii().as_str_ascii()); + + let search_idx = lookahead.position_elem(&self.boundary[0]) + .unwrap_or(lookahead.len() - 1); + + debug!("Search idx: {}", search_idx); + + self.boundary_read = lookahead[search_idx..] + .starts_with(self.boundary[]); + + debug!("Boundary read: {} Boundary: {}", self.boundary_read, self.boundary.to_ascii().as_str_ascii()); + + self.last_search_idx += search_idx; + + if !self.boundary_read { + self.last_search_idx += 1; + } - Ok(lookahead[self.last_search_idx..].starts_with(self.boundary[])) } else if self.last_search_idx == 0 { - Err(standard_error(EndOfFile)) - } else { Ok(true) } + return Err(standard_error(EndOfFile)) + } + + Ok(()) } - fn consume_boundary(&mut self) { - self.reader.consume(self.last_search_idx + self.boundary.len()); + /// Read bytes until the reader is full + fn true_fill_buf(&mut self) -> IoResult<()> { + debug!("True fill buf! Buf len: {}", self.buf_len); + + let mut bytes_read = 1u; + + while bytes_read != 0 { + debug!("Bytes read loop!"); + + bytes_read = match self.reader.read(self.buf[mut self.buf_len..]) { + Ok(read) => read, + Err(err) => if err.kind == EndOfFile { break; } else { return Err(err); }, + }; + + debug!("Bytes read: {}", bytes_read); + + self.buf_len += bytes_read; + } + + debug!("Exited bytes read loop!"); + + Ok(()) + } + + fn _consume(&mut self, amt: uint) { + use std::ptr::copy_memory; + + debug!("Consume! Amt: {}", amt); + + assert!(amt <= self.buf_len); + + let src = self.buf[amt..].as_ptr(); + let dest = self.buf[mut].as_mut_ptr(); + + unsafe { copy_memory(dest, src, self.buf_len - amt); } + + self.buf_len -= amt; + self.last_search_idx -= amt; + } + + fn consume_boundary(&mut self) -> IoResult<()> { + debug!("Consume boundary!"); + + while !self.boundary_read { + debug!("Boundary read loop!"); + + match self.read_to_boundary() { + Ok(_) => (), + Err(e) => if e.kind == EndOfFile { + break; + } else { + return Err(e); + } + } + } + + let consume_amt = cmp::min(self.buf_len, self.last_search_idx + self.boundary.len()); + + debug!("Consume amt: {} Buf len: {}", consume_amt, self.buf_len); + + self._consume(consume_amt); self.last_search_idx = 0; - self.boundary_read = false; + self.boundary_read = false; + + Ok(()) } + #[allow(unused)] fn set_boundary(&mut self, boundary: String) { self.boundary = boundary.into_bytes(); } } -impl Reader for BoundaryReader { +impl Reader for BoundaryReader where S: Reader { fn read(&mut self, buf: &mut [u8]) -> IoResult { use std::cmp; + use std::slice::bytes::copy_memory; - self.boundary_read = try!(self.read_to_boundary()); + debug!("Read!"); - let trunc_len = cmp::min(buf.len(), self.last_search_idx); + try!(self.read_to_boundary()); - let trunc_buf = buf[mut ..trunc_len]; // Truncate the buffer so we don't read ahead + let trunc_len = cmp::min(buf.len(), self.last_search_idx); + copy_memory(buf, self.buf[..trunc_len]); - let bytes_read = self.reader.read(trunc_buf).unwrap(); - self.last_search_idx -= bytes_read; + self._consume(trunc_len); - Ok(bytes_read) + Ok(trunc_len) } } -trait Prepend { - fn prepend(&mut self, t: T); -} +impl Buffer for BoundaryReader where S: Reader { + fn fill_buf<'a>(&'a mut self) -> IoResult<&'a [u8]> { + debug!("Fill buf!"); + + try!(self.read_to_boundary()); + + let buf = self.buf[..self.last_search_idx]; + + debug!("Buf: {}", buf.to_ascii().as_str_ascii()); -impl Prepend for String { - fn prepend(&mut self, s: S) { - unsafe { - self.as_mut_vec().prepend(s.as_slice().as_bytes()); - } + Ok(buf) + } + + fn consume(&mut self, amt: uint) { + assert!(amt <= self.last_search_idx); + self._consume(amt); } } -impl<'a, T> Prepend<&'a [T]> for Vec { - fn prepend(&mut self, slice: &[T]) { - use std::ptr::copy_memory; +#[test] +fn test_boundary() { + use std::io::BufReader; - let old_len = self.len(); + const BOUNDARY: &'static str = "--boundary\r\n"; + const TEST_VAL: &'static str = "\r +--boundary\r +dashed-value-1\r +--boundary\r +dashed-value-2\r +--boundary\r +"; - self.reserve(slice.len()); + let test_reader = BufReader::new(TEST_VAL.as_bytes()); + let mut reader = BoundaryReader::from_reader(test_reader, BOUNDARY.into_string()); - unsafe { - self.set_len(old_len + slice.len()); - copy_memory(self[mut slice.len()..].as_mut_ptr(), self[..old_len].as_ptr(), old_len); - copy_memory(self.as_mut_ptr(), slice.as_ptr(), slice.len()); - } - } -} + debug!("Read 1"); + let string = reader.read_to_string().unwrap(); + debug!("{}", string); + assert!(string[].trim().is_empty()); -#[test] -fn test_prepend() { - let mut vec = vec![3u64, 4, 5]; - vec.prepend([1u64, 2]); - assert_eq!(vec[], [1u64, 2, 3, 4, 5][]); -} + debug!("Consume 1"); + reader.consume_boundary().unwrap(); -#[test] -fn test_prepend_string() { - let mut string = "World!".into_string(); - string.prepend("Hello, "); - assert_eq!(&*string, "Hello, World!"); + debug!("Read 2"); + assert_eq!(reader.read_to_string().unwrap()[].trim(), "dashed-value-1"); + + debug!("Consume 2"); + reader.consume_boundary().unwrap(); + + debug!("Read 3"); + assert_eq!(reader.read_to_string().unwrap()[].trim(), "dashed-value-2"); + + debug!("Consume 3"); + reader.consume_boundary().unwrap(); + } From e7902aef7458055ec09e08893528ea036f0188f4 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Thu, 20 Nov 2014 00:30:50 -0800 Subject: [PATCH 011/453] Fix compiler errors --- multipart/src/client.rs | 18 +++++++++--------- multipart/src/lib.rs | 10 +++++----- multipart/src/server.rs | 22 ++++++++++++---------- 3 files changed, 26 insertions(+), 24 deletions(-) diff --git a/multipart/src/client.rs b/multipart/src/client.rs index 26bc98069..2a51ae16c 100644 --- a/multipart/src/client.rs +++ b/multipart/src/client.rs @@ -3,7 +3,7 @@ use hyper::header::common::ContentType; use hyper::net::{Fresh, Streaming}; use hyper::{HttpResult, HttpIoError}; -use mime::{mod, Mime}; +use mime::{Mime, TopLevel, SubLevel, Attr, Value}; use mime_guess::guess_mime_type; @@ -11,7 +11,7 @@ use std::io::IoResult; use std::io::fs::File; use std::io; -use super::{MultipartField, TextField, FileField, MultipartFile}; +use super::{MultipartField, MultipartFile}; const BOUNDARY_LEN: uint = 8; @@ -33,7 +33,7 @@ impl<'a> Multipart<'a> { } pub fn add_text(&mut self, name: &str, val: &str) { - self.fields.push((name.into_string(), TextField(val.into_string()))); + self.fields.push((name.into_string(), MultipartField::Text(val.into_string()))); } /// Add the file to the multipart request, guessing its `Content-Type` from its extension @@ -42,7 +42,7 @@ impl<'a> Multipart<'a> { let content_type = guess_mime_type(file.path()); self.fields.push((name.into_string(), - FileField(MultipartFile::from_file(filename, file, content_type)))); + MultipartField::File(MultipartFile::from_file(filename, file, content_type)))); } /// Apply the appropriate headers to the `Request` and send the data. @@ -71,9 +71,9 @@ impl<'a> Multipart<'a> { try!(write!(req, "Content-Disposition: form-data; name=\"{}\"", name)); try!(match field { - TextField(text) => req.write(b"\r\n\r\n") + MultipartField::Text(text) => req.write(b"\r\n\r\n") .and_then(|_| write_line(req, &*text)), // Style suggestions welcome - FileField(file) => write_file(req, file), + MultipartField::File(file) => write_file(req, file), }); try!(write_boundary(req, boundary[])); @@ -111,9 +111,9 @@ fn io_to_http(res: IoResult) -> HttpResult { } fn multipart_mime(bound: &str) -> Mime { - mime::Mime( - mime::Multipart, mime::SubExt("form-data".into_string()), - vec![(mime::AttrExt("boundary".into_string()), mime::ValueExt(bound.into_string()))] + Mime( + TopLevel::Multipart, SubLevel::Ext("form-data".into_string()), + vec![(Attr::Ext("boundary".into_string()), Value::Ext(bound.into_string()))] ) } diff --git a/multipart/src/lib.rs b/multipart/src/lib.rs index 9c18b09f8..d31ade488 100644 --- a/multipart/src/lib.rs +++ b/multipart/src/lib.rs @@ -1,4 +1,4 @@ -#![feature(if_let, slicing_syntax, default_type_params, phase, macro_rules)] +#![feature(if_let, slicing_syntax, default_type_params, phase, unboxed_closures, macro_rules)] extern crate hyper; #[phase(plugin, link)] extern crate log; @@ -45,8 +45,8 @@ impl<'a> MultipartFile<'a> { pub enum MultipartField<'a> { - TextField(String), - FileField(MultipartFile<'a>), + Text(String), + File(MultipartFile<'a>), // MultiFiles(Vec), /* TODO: Multiple files */ } @@ -58,7 +58,7 @@ impl<'a> Reader for MultipartFile<'a> { pub mod mime_guess { - use mime::{mod, Mime}; + use mime::{Mime, TopLevel, SubLevel}; use serialize::json; @@ -120,7 +120,7 @@ pub mod mime_guess { } pub fn octet_stream() -> Mime { - Mime(mime::Application, mime::SubExt("octet-stream".into_string()), Vec::new()) + Mime(TopLevel::Application, SubLevel::Ext("octet-stream".into_string()), Vec::new()) } #[test] diff --git a/multipart/src/server.rs b/multipart/src/server.rs index e7eb5d011..bff2fdd0f 100644 --- a/multipart/src/server.rs +++ b/multipart/src/server.rs @@ -1,21 +1,21 @@ use hyper::header::common::content_type::ContentType; use hyper::server::request::Request; -use mime::{Mime, AttrExt, ValueExt}; +use mime::{Mime, TopLevel, SubLevel, Attr, Value}; -use super::{MultipartField, TextField, MultipartFile, FileField}; +use super::{MultipartField, MultipartFile}; use std::io::{RefReader, BufferedReader, IoError, IoResult, EndOfFile, standard_error, OtherIoError}; use std::kinds::marker; fn is_multipart_formdata(req: &Request) -> bool { - use mime::{Multipart, Mime, SubExt}; + use mime::{Multipart}; req.headers.get::().map(|ct| { let ContentType(ref mime) = *ct; match *mime { - Mime(Multipart, SubExt(ref subtype), _) => subtype[] == "form-data", + Mime(TopLevel::Multipart, SubLevel::Ext(ref subtype), _) => subtype[] == "form-data", _ => false, } }).unwrap_or(false) @@ -26,11 +26,11 @@ fn get_boundary(ct: &ContentType) -> Option { let Mime(_, _, ref params) = *mime; params.iter().find(|&&(ref name, _)| - if let AttrExt(ref name) = *name { + if let Attr::Ext(ref name) = *name { name[] == "boundary" } else { false } ).and_then(|&(_, ref val)| - if let ValueExt(ref val) = *val { + if let Value::Ext(ref val) = *val { Some(val.clone()) } else { None } ) @@ -72,11 +72,13 @@ impl Multipart { if let Some(content_type) = try!(self.read_content_type()) { let _ = try!(self.source.reader.read_line()); // Consume empty line - Ok((field_name, FileField(MultipartFile::from_octet(filename, &mut self.source, content_type[])))) + Ok((field_name, MultipartField::File( + MultipartFile::from_octet(filename, &mut self.source, content_type[]))) + ) } else { // Empty line consumed by read_content_type() let text = try!(self.source.read_to_string()); - Ok((field_name, TextField(text))) + Ok((field_name, MultipartField::Text(text))) } } @@ -84,7 +86,7 @@ impl Multipart { /// This is a substitute for `Multipart` implementing `Iterator`, /// since `Iterator::next()` can't use bound lifetimes. /// See https://www.reddit.com/r/rust/comments/2lkk4i/concrete_lifetime_vs_bound_lifetime/ - pub fn foreach_entry FnMut(String, MultipartField<'a>)>(&mut self, f: F) { + pub fn foreach_entry FnMut(String, MultipartField<'a>)>(&mut self, mut f: F) { loop { match self.read_entry() { Ok((name, field)) => f(name, field), @@ -277,7 +279,7 @@ impl<'a, T> Prepend<&'a [T]> for Vec { #[test] fn test_prepend() { let mut vec = vec![3u64, 4, 5]; - vec.prepend([1u64, 2]); + vec.prepend(&[1u64, 2]); assert_eq!(vec[], [1u64, 2, 3, 4, 5][]); } From 8c81a86dc1be8506d1c74bd17023345ac021041e Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Thu, 20 Nov 2014 01:13:40 -0800 Subject: [PATCH 012/453] Compiler ICE --- multipart/src/bin/multipart_server.rs | 4 +++- multipart/src/lib.rs | 5 +---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/multipart/src/bin/multipart_server.rs b/multipart/src/bin/multipart_server.rs index 17fa556a5..fb73a3d10 100644 --- a/multipart/src/bin/multipart_server.rs +++ b/multipart/src/bin/multipart_server.rs @@ -1,3 +1,5 @@ +#![feature(unboxed_closures)] + extern crate hyper; extern crate multipart; @@ -14,7 +16,7 @@ fn hello(mut incoming: Incoming) { let mut multipart = Multipart::from_request(req).ok().expect("Could not create multipart!"); - multipart.foreach_entry(|name, content| println!("Name: {} Content: {}", name, content)); + multipart.foreach_entry(|&mut: name, content| println!("Name: {} Content: {}", name, content)); *res.status_mut() = status::Ok; diff --git a/multipart/src/lib.rs b/multipart/src/lib.rs index 9ebaa62ab..9d5dd123b 100644 --- a/multipart/src/lib.rs +++ b/multipart/src/lib.rs @@ -19,10 +19,9 @@ pub mod server; pub struct MultipartFile<'a> { - _marker: marker::ContravariantLifetime<'a>, filename: Option, content_type: Mime, - reader: &'a mut Reader +'a , + reader: &'a mut Reader + 'a, } impl<'a> MultipartFile<'a> { @@ -31,7 +30,6 @@ impl<'a> MultipartFile<'a> { filename: filename, reader: reader, content_type: from_str(cont_type).unwrap_or_else(mime_guess::octet_stream), - _marker: marker::ContravariantLifetime, } } @@ -40,7 +38,6 @@ impl<'a> MultipartFile<'a> { filename: filename, reader: reader, content_type: mime, - _marker: marker::ContravariantLifetime, } } } From 27211eb0dba42f09610c81842030c875e39ee342 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sun, 23 Nov 2014 17:06:33 -0800 Subject: [PATCH 013/453] Fix compiler issues, have client::Multipart send sized requests --- multipart/src/bin/multipart_server.rs | 55 +++++++++---- multipart/src/client.rs | 113 ++++++++++++++++++-------- multipart/src/lib.rs | 5 +- multipart/src/server.rs | 21 +++-- 4 files changed, 138 insertions(+), 56 deletions(-) diff --git a/multipart/src/bin/multipart_server.rs b/multipart/src/bin/multipart_server.rs index fb73a3d10..df8258769 100644 --- a/multipart/src/bin/multipart_server.rs +++ b/multipart/src/bin/multipart_server.rs @@ -1,31 +1,56 @@ -#![feature(unboxed_closures)] +#![feature(unboxed_closures, if_let, slicing_syntax)] extern crate hyper; extern crate multipart; -use self::hyper::server::{Server, Incoming}; +use self::hyper::server::{Server, Request, Response}; use self::hyper::status; use self::multipart::server::Multipart; use std::io::net::ip::Ipv4Addr; -fn hello(mut incoming: Incoming) { - for connection in incoming { - let (mut req, mut res) = connection.open().ok().expect("Connection failed!"); - - let mut multipart = Multipart::from_request(req).ok().expect("Could not create multipart!"); +fn hello(req: Request, mut res: Response) { + print_req(&req); + + let mut multipart = Multipart::from_request(req).ok().expect("Could not create multipart!"); - multipart.foreach_entry(|&mut: name, content| println!("Name: {} Content: {}", name, content)); - - *res.status_mut() = status::Ok; + multipart.foreach_entry(|&: name, content| println!("Name: {} Content: {}", name, content)); + + *res.status_mut() = status::Ok; - res.start().unwrap().end().unwrap(); - } + res.start().unwrap().end().unwrap(); } fn main() { - let server = Server::http(Ipv4Addr(127, 0, 0, 1), 1337); - server.listen(hello).unwrap(); - + let args = std::os::args(); + + if args.iter().find(|s| "tcp"== s[]).is_some() { + tcp_listen(); + } else { + let server = Server::http(Ipv4Addr(127, 0, 0, 1), 1337); + server.listen(hello).unwrap(); + } +} + + + +fn tcp_listen() { + use std::io::{Acceptor, Listener, TcpListener}; + use std::io::util::copy; + + let tcp = TcpListener::bind(("localhost", 1337u16)).unwrap(); + + let ref mut stdout = std::io::stdout(); + + for conn in tcp.listen().unwrap().incoming() { + let ref mut conn = conn.unwrap(); + + copy(conn, stdout).unwrap(); + } +} + +fn print_req(req: &Request) { + println!("Reqeust: \nRemote addr: {}\nMethod: {}\nHeaders: {}\nURI: {}", + req.remote_addr, req.method, req.headers, req.uri); } diff --git a/multipart/src/client.rs b/multipart/src/client.rs index 02296ae5e..6bb740abe 100644 --- a/multipart/src/client.rs +++ b/multipart/src/client.rs @@ -1,5 +1,7 @@ use hyper::client::{Request, Response}; -use hyper::header::common::ContentType; + +use hyper::header::common::{ContentType, ContentLength}; + use hyper::net::{Fresh, Streaming}; use hyper::{HttpResult, HttpIoError}; @@ -15,9 +17,14 @@ use super::{MultipartField, MultipartFile}; const BOUNDARY_LEN: uint = 8; +type Fields<'a> = Vec<(String, MultipartField<'a>)>; + pub struct Multipart<'a> { - fields: Vec<(String, MultipartField<'a>)>, + fields: Fields<'a>, boundary: String, + /// If the request can be sized. + /// If true, avoid using chunked requests. + pub sized: bool, } /// Shorthand for a writable request (`Request`) @@ -29,6 +36,7 @@ impl<'a> Multipart<'a> { Multipart { fields: Vec::new(), boundary: random_alphanumeric(BOUNDARY_LEN), + sized: false, } } @@ -46,60 +54,87 @@ impl<'a> Multipart<'a> { } /// Apply the appropriate headers to the `Request` and send the data. + /// If `self.sized == true`, send a sized (non-chunked) request, setting the `Content-Length` + /// header. Else, send a chunked request. pub fn send(self, mut req: Request) -> HttpResult { use hyper::method; assert!(req.method() == method::Post, "Multipart request must use POST method!"); - self.apply_headers(&mut req); - debug!("Fields: {}; Boundary: {}", self.fields[], self.boundary[]); - debug!("{}", req.headers()); + if self.sized { + return self.send_sized(req); + } + + let Multipart { fields, boundary, sized } = self; + apply_headers(&mut req, boundary[], None); + + debug!("{}", req.headers()); + let mut req = try!(req.start()); - try!(io_to_http(self.write_request(&mut req))); + try!(io_to_http(write_body(&mut req, fields, boundary[]))); req.send() } - - fn apply_headers(&self, req: &mut Request){ - let headers = req.headers_mut(); + + fn send_sized(self, mut req: Request) -> HttpResult { + let mut body: Vec = Vec::new(); - headers.set(ContentType(multipart_mime(self.boundary[]))) - } + let Multipart { fields, boundary, sized } = self; - fn write_request(self, req: &mut ReqWrite) -> IoResult<()> { - let Multipart{ fields, boundary } = self; + try!(write_body(&mut body, fields, boundary[])); + + apply_headers(&mut req, boundary[], Some(body.len())); + + let mut req = try!(req.start()); + try!(io_to_http(req.write(body[]))); + req.send() + } +} - try!(write_boundary(req, boundary[])); +fn apply_headers(req: &mut Request, boundary: &str, size: Option){ + let headers = req.headers_mut(); - for (name, field) in fields.into_iter() { - try!(write!(req, "Content-Disposition: form-data; name=\"{}\"\r\n\r\n", name)); + headers.set(ContentType(multipart_mime(boundary))); - try!(match field { - MultipartField::Text(text) => write_line(req, &*text), - MultipartField::File(file) => write_file(req, file), - }); - - try!(write_boundary(req, boundary[])); - } + if let Some(size) = size { + headers.set(ContentLength(size)); + } +} - Ok(()) +fn write_body<'a>(wrt: &mut Writer, fields: Fields<'a>, boundary: &str) -> IoResult<()> { + try!(write_boundary(wrt, boundary[])); + + for (name, field) in fields.into_iter() { + try!(write_field(wrt, name, field, boundary)); } -} + Ok(()) +} + +fn write_field(wrt: &mut Writer, name: String, field: MultipartField, boundary: &str) -> IoResult<()> { + try!(write!(wrt, "Content-Disposition: form-data; name=\"{}\"\r\n\r\n", name)); -fn write_boundary(req: &mut ReqWrite, boundary: &str) -> IoResult<()> { - write!(req, "--{}\r\n", boundary) + try!(match field { + MultipartField::Text(text) => write_line(wrt, &*text), + MultipartField::File(file) => write_file(wrt, file), + }); + + write_boundary(wrt, boundary[]) +} + +fn write_boundary(wrt: &mut Writer, boundary: &str) -> IoResult<()> { + write!(wrt, "--{}\r\n", boundary) } -fn write_file(req: &mut ReqWrite, mut file: MultipartFile) -> IoResult<()> { - try!(file.filename.map(|filename| write!(req, "; filename=\"{}\"\r\n", filename)).unwrap_or(Ok(()))); - try!(write!(req, "Content-Type: {}\r\n\r\n", file.content_type)); - io::util::copy(&mut file.reader, req) +fn write_file(wrt: &mut Writer, mut file: MultipartFile) -> IoResult<()> { + try!(file.filename.map(|filename| write!(wrt, "; filename=\"{}\"\r\n", filename)).unwrap_or(Ok(()))); + try!(write!(wrt, "Content-Type: {}\r\n\r\n", file.content_type)); + ref_copy(&mut file.reader, wrt) } /// Specialized write_line that writes CRLF after a line as per W3C specs -fn write_line(req: &mut ReqWrite, s: &str) -> IoResult<()> { +fn write_line(req: &mut Writer, s: &str) -> IoResult<()> { req.write_str(s).and_then(|_| req.write(b"\r\n")) } @@ -122,5 +157,17 @@ fn multipart_mime(bound: &str) -> Mime { ) } - +/// A copy of std::io::util::copy that takes references +fn ref_copy(r: &mut Reader, w: &mut Writer) -> IoResult<()> { + let mut buf = [0, ..1024 * 64]; + + loop { + let len = match r.read(&mut buf) { + Ok(len) => len, + Err(ref e) if e.kind == io::EndOfFile => return Ok(()), + Err(e) => return Err(e), + }; + try!(w.write(buf[..len])); + } +} diff --git a/multipart/src/lib.rs b/multipart/src/lib.rs index 9d5dd123b..af0985f2a 100644 --- a/multipart/src/lib.rs +++ b/multipart/src/lib.rs @@ -7,7 +7,9 @@ extern crate serialize; use self::mime::Mime; -use std::fmt::{Formatter, FormatError, Show}; +use std::fmt::{Formatter, Show}; +use std::fmt::Error as FormatError; + use std::kinds::marker; use std::io::fs::File; use std::io::{AsRefReader, RefReader, IoResult}; @@ -151,6 +153,7 @@ mod test { multipart.add_text("hello", "world"); multipart.add_text("goodnight", "sun"); + multipart.sized = true; multipart.send(request).unwrap(); } diff --git a/multipart/src/server.rs b/multipart/src/server.rs index 5ab392eb3..073286cc7 100644 --- a/multipart/src/server.rs +++ b/multipart/src/server.rs @@ -14,13 +14,16 @@ use std::kinds::marker; fn is_multipart_formdata(req: &Request) -> bool { use mime::{Multipart}; - req.headers.get::().map(|ct| { + req.headers.get::().map_or(false, |ct| { let ContentType(ref mime) = *ct; + + debug!("Content-Type: {}", mime); + match *mime { - Mime(TopLevel::Multipart, SubLevel::Ext(ref subtype), _) => subtype[] == "form-data", + Mime(TopLevel::Multipart, SubLevel::FormData, _) => true, _ => false, } - }).unwrap_or(false) + }) } fn get_boundary(ct: &ContentType) -> Option { @@ -38,8 +41,8 @@ fn get_boundary(ct: &ContentType) -> Option { ) } -pub struct Multipart { - source: BoundaryReader, +pub struct Multipart<'a> { + source: BoundaryReader>, } macro_rules! try_find( @@ -48,18 +51,22 @@ macro_rules! try_find( ) ) -impl Multipart { +impl<'a> Multipart<'a> { /// If the given `Request` is of `Content-Type: multipart/form-data`, return /// the wrapped request as `Ok(Multipart)`, otherwise `Err(Request)`. - pub fn from_request(mut req: Request) -> Result { + pub fn from_request(mut req: Request<'a>) -> Result, Request<'a>> { use std::io::stdio::stdout_raw; if !is_multipart_formdata(&req) { return Err(req); } + debug!("Is multipart!"); + let boundary = if let Some(boundary) = req.headers.get::() .and_then(get_boundary) { boundary } else { return Err(req); }; + debug!("Boundary: {}", boundary); + Ok(Multipart { source: BoundaryReader::from_reader(req, format!("--{}\r\n", boundary)) }) } From d1b5b554378132fd9e62baa91a4a4db2ad02f8ae Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Mon, 24 Nov 2014 02:37:37 -0800 Subject: [PATCH 014/453] Tests pass! --- multipart/src/lib.rs | 2 +- multipart/src/server.rs | 13 +------------ 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/multipart/src/lib.rs b/multipart/src/lib.rs index af0985f2a..1cc9a08b8 100644 --- a/multipart/src/lib.rs +++ b/multipart/src/lib.rs @@ -153,7 +153,7 @@ mod test { multipart.add_text("hello", "world"); multipart.add_text("goodnight", "sun"); - multipart.sized = true; + //multipart.sized = true; multipart.send(request).unwrap(); } diff --git a/multipart/src/server.rs b/multipart/src/server.rs index 073286cc7..eb666da08 100644 --- a/multipart/src/server.rs +++ b/multipart/src/server.rs @@ -72,18 +72,7 @@ impl<'a> Multipart<'a> { pub fn read_entry<'a>(&'a mut self) -> IoResult<(String, MultipartField<'a>)> { debug!("Read entry!"); - - loop { - let string = self.source.read_to_string().unwrap(); - self.source.consume_boundary(); - - if string.is_empty() { - break; - } - - debug!("{}", string); - } - + try!(self.source.consume_boundary()); let (disp_type, field_name, filename) = try!(self.read_content_disposition()); From 1b8f911a63c5d9cc09ff47d9bbbdf934af876db3 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Mon, 24 Nov 2014 03:11:47 -0800 Subject: [PATCH 015/453] Fix compiler warnings, finish work --- multipart/src/bin/multipart_server.rs | 3 +- multipart/src/client.rs | 27 +----- multipart/src/lib.rs | 118 ++++++++++---------------- multipart/src/mime_guess.rs | 73 ++++++++++++++++ multipart/src/server.rs | 8 +- 5 files changed, 124 insertions(+), 105 deletions(-) create mode 100644 multipart/src/mime_guess.rs diff --git a/multipart/src/bin/multipart_server.rs b/multipart/src/bin/multipart_server.rs index df8258769..af6dea745 100644 --- a/multipart/src/bin/multipart_server.rs +++ b/multipart/src/bin/multipart_server.rs @@ -1,4 +1,5 @@ #![feature(unboxed_closures, if_let, slicing_syntax)] +#![allow(dead_code)] extern crate hyper; extern crate multipart; @@ -51,6 +52,6 @@ fn tcp_listen() { } fn print_req(req: &Request) { - println!("Reqeust: \nRemote addr: {}\nMethod: {}\nHeaders: {}\nURI: {}", + println!("Request: \nRemote addr: {}\nMethod: {}\nHeaders: {}\nURI: {}", req.remote_addr, req.method, req.headers, req.uri); } diff --git a/multipart/src/client.rs b/multipart/src/client.rs index 6bb740abe..3b66c08da 100644 --- a/multipart/src/client.rs +++ b/multipart/src/client.rs @@ -11,9 +11,8 @@ use mime_guess::guess_mime_type; use std::io::IoResult; use std::io::fs::File; -use std::io; -use super::{MultipartField, MultipartFile}; +use super::{MultipartField, MultipartFile, ref_copy, random_alphanumeric}; const BOUNDARY_LEN: uint = 8; @@ -66,7 +65,7 @@ impl<'a> Multipart<'a> { return self.send_sized(req); } - let Multipart { fields, boundary, sized } = self; + let Multipart { fields, boundary, ..} = self; apply_headers(&mut req, boundary[], None); @@ -80,7 +79,7 @@ impl<'a> Multipart<'a> { fn send_sized(self, mut req: Request) -> HttpResult { let mut body: Vec = Vec::new(); - let Multipart { fields, boundary, sized } = self; + let Multipart { fields, boundary, ..} = self; try!(write_body(&mut body, fields, boundary[])); @@ -138,13 +137,6 @@ fn write_line(req: &mut Writer, s: &str) -> IoResult<()> { req.write_str(s).and_then(|_| req.write(b"\r\n")) } -/// Generate a random alphanumeric sequence of length `len` -fn random_alphanumeric(len: uint) -> String { - use std::rand::{task_rng, Rng}; - use std::char::to_lowercase; - - task_rng().gen_ascii_chars().map(to_lowercase).take(len).collect() -} fn io_to_http(res: IoResult) -> HttpResult { res.map_err(|e| HttpIoError(e)) @@ -157,17 +149,4 @@ fn multipart_mime(bound: &str) -> Mime { ) } -/// A copy of std::io::util::copy that takes references -fn ref_copy(r: &mut Reader, w: &mut Writer) -> IoResult<()> { - let mut buf = [0, ..1024 * 64]; - - loop { - let len = match r.read(&mut buf) { - Ok(len) => len, - Err(ref e) if e.kind == io::EndOfFile => return Ok(()), - Err(e) => return Err(e), - }; - try!(w.write(buf[..len])); - } -} diff --git a/multipart/src/lib.rs b/multipart/src/lib.rs index 1cc9a08b8..11cab64e7 100644 --- a/multipart/src/lib.rs +++ b/multipart/src/lib.rs @@ -10,15 +10,13 @@ use self::mime::Mime; use std::fmt::{Formatter, Show}; use std::fmt::Error as FormatError; -use std::kinds::marker; -use std::io::fs::File; -use std::io::{AsRefReader, RefReader, IoResult}; +use std::io::{File, IoErrorKind, IoResult}; -use server::BoundaryReader; +use std::io::fs::PathExtensions; pub mod client; pub mod server; - +pub mod mime_guess; pub struct MultipartFile<'a> { filename: Option, @@ -42,6 +40,33 @@ impl<'a> MultipartFile<'a> { content_type: mime, } } + + /// Save this file to `path`, ignoring the filename, if any. + /// + /// Returns the created file on success. + pub fn save_as(&mut self, path: &Path) -> IoResult { + let mut file = try!(File::create(path)); + + try!(ref_copy(self.reader, &mut file)); + + Ok(file) + } + + /// Save this file in the directory described by `dir`, + /// appending `filename` if any, or a random string. + /// + /// Returns the created file on success. + /// + /// ###Panics + /// If `dir` does not represent a directory. + pub fn save_in(&mut self, dir: &Path) -> IoResult { + assert!(dir.is_dir(), "Given path is not a directory!"); + + let filename = self.filename.as_ref().map_or_else(|| random_alphanumeric(10), |s| s.clone()); + let path = dir.join(filename); + + self.save_as(&path) + } } impl<'a> Show for MultipartFile<'a> { @@ -63,80 +88,25 @@ impl<'a> Reader for MultipartFile<'a> { } } -pub mod mime_guess { - - use mime::{Mime, TopLevel, SubLevel}; - - use serialize::json; +/// A copy of `std::io::util::copy` that takes trait references +pub fn ref_copy(r: &mut Reader, w: &mut Writer) -> IoResult<()> { + let mut buf = [0, ..1024 * 64]; - use std::cell::RefCell; - - use std::collections::HashMap; - - - /// Guess the MIME type of the `Path` by its extension. - /// - /// **Guess** is the operative word here, as the contents of a file - /// may not or may not match its MIME type/extension. - pub fn guess_mime_type(path: &Path) -> Mime { - let ext = path.extension_str().unwrap_or(""); - - get_mime_type(ext) - } - - pub fn guess_mime_type_filename(filename: &str) -> Mime { - let path = Path::new(filename); - - guess_mime_type(&path) - } - - local_data_key!(mime_types_cache: RefCell>) - - /// Get the MIME type associated with a file extension - // MIME Types are cached in a task-local heap - pub fn get_mime_type(ext: &str) -> Mime { - if ext.is_empty() { return octet_stream(); } - - let ext = ext.into_string(); - - let cache = if let Some(cache) = mime_types_cache.get() { cache } - else { - mime_types_cache.replace(Some(RefCell::new(HashMap::new()))); - mime_types_cache.get().unwrap() + loop { + let len = match r.read(&mut buf) { + Ok(len) => len, + Err(ref e) if e.kind == IoErrorKind::EndOfFile => return Ok(()), + Err(e) => return Err(e), }; - - if let Some(mime_type) = cache.borrow().find(&ext) { - return mime_type.clone(); - } - - let mime_type = find_mime_type(&*ext); - - cache.borrow_mut().insert(ext, mime_type.clone()); - - mime_type - } - - const MIME_TYPES: &'static str = include_str!("../mime_types.json"); - - /// Load the MIME_TYPES as JSON and try to locate `ext` - fn find_mime_type(ext: &str) -> Mime { - json::from_str(MIME_TYPES).unwrap() - .find(ext).and_then(|j| j.as_string()) - .and_then(from_str::) - .unwrap_or_else(octet_stream) + try!(w.write(buf[..len])); } +} - pub fn octet_stream() -> Mime { - Mime(TopLevel::Application, SubLevel::Ext("octet-stream".into_string()), Vec::new()) - } +/// Generate a random alphanumeric sequence of length `len` +fn random_alphanumeric(len: uint) -> String { + use std::rand::{task_rng, Rng}; -#[test] - fn test_mime_type_guessing() { - assert!(get_mime_type("gif").to_string() == "image/gif".to_string()); - assert!(get_mime_type("txt").to_string() == "text/plain".to_string()); - assert!(get_mime_type("blahblah").to_string() == "application/octet-stream".to_string()); - } - + task_rng().gen_ascii_chars().map(|ch| ch.to_lowercase()).take(len).collect() } #[cfg(test)] diff --git a/multipart/src/mime_guess.rs b/multipart/src/mime_guess.rs new file mode 100644 index 000000000..9dfa7b400 --- /dev/null +++ b/multipart/src/mime_guess.rs @@ -0,0 +1,73 @@ +use mime::{Mime, TopLevel, SubLevel}; + +use serialize::json; + +use std::cell::RefCell; + +use std::collections::HashMap; + + +/// Guess the MIME type of the `Path` by its extension. +/// +/// **Guess** is the operative word here, as the contents of a file +/// may not or may not match its MIME type/extension. +pub fn guess_mime_type(path: &Path) -> Mime { + let ext = path.extension_str().unwrap_or(""); + + get_mime_type(ext) +} + +pub fn guess_mime_type_filename(filename: &str) -> Mime { + let path = Path::new(filename); + + guess_mime_type(&path) +} + +local_data_key!(mime_types_cache: RefCell>) + +/// Get the MIME type associated with a file extension +// MIME Types are cached in a task-local heap +pub fn get_mime_type(ext: &str) -> Mime { + if ext.is_empty() { return octet_stream(); } + + let ext = ext.into_string(); + + let cache = if let Some(cache) = mime_types_cache.get() { cache } + else { + mime_types_cache.replace(Some(RefCell::new(HashMap::new()))); + mime_types_cache.get().unwrap() + }; + + if let Some(mime_type) = cache.borrow().get(&ext) { + return mime_type.clone(); + } + + let mime_type = find_mime_type(&*ext); + + cache.borrow_mut().insert(ext, mime_type.clone()); + + mime_type +} + +const MIME_TYPES: &'static str = include_str!("../mime_types.json"); + +/// Load the MIME_TYPES as JSON and try to locate `ext` +fn find_mime_type(ext: &str) -> Mime { + json::from_str(MIME_TYPES).unwrap() + .find(ext).and_then(|j| j.as_string()) + .and_then(from_str::) + .unwrap_or_else(octet_stream) +} + +pub fn octet_stream() -> Mime { + Mime(TopLevel::Application, SubLevel::Ext("octet-stream".into_string()), Vec::new()) +} + +#[test] +fn test_mime_type_guessing() { + assert!(get_mime_type("gif").to_string() == "image/gif".to_string()); + assert!(get_mime_type("txt").to_string() == "text/plain".to_string()); + assert!(get_mime_type("blahblah").to_string() == "application/octet-stream".to_string()); +} + + diff --git a/multipart/src/server.rs b/multipart/src/server.rs index eb666da08..dab929542 100644 --- a/multipart/src/server.rs +++ b/multipart/src/server.rs @@ -7,9 +7,7 @@ use super::{MultipartField, MultipartFile}; use std::cmp; -use std::io::{RefReader, BufferedReader, IoError, IoResult, EndOfFile, standard_error, OtherIoError}; - -use std::kinds::marker; +use std::io::{IoError, IoResult, EndOfFile, standard_error, OtherIoError}; fn is_multipart_formdata(req: &Request) -> bool { use mime::{Multipart}; @@ -55,9 +53,7 @@ impl<'a> Multipart<'a> { /// If the given `Request` is of `Content-Type: multipart/form-data`, return /// the wrapped request as `Ok(Multipart)`, otherwise `Err(Request)`. - pub fn from_request(mut req: Request<'a>) -> Result, Request<'a>> { - use std::io::stdio::stdout_raw; - + pub fn from_request(req: Request<'a>) -> Result, Request<'a>> { if !is_multipart_formdata(&req) { return Err(req); } debug!("Is multipart!"); From 1a43df84a06da153aadbdbac59e3b9d1bc44bdbc Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Mon, 24 Nov 2014 03:18:33 -0800 Subject: [PATCH 016/453] cleanup --- multipart/src/bin/multipart_server.rs | 2 -- multipart/src/lib.rs | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/multipart/src/bin/multipart_server.rs b/multipart/src/bin/multipart_server.rs index af6dea745..59d9de022 100644 --- a/multipart/src/bin/multipart_server.rs +++ b/multipart/src/bin/multipart_server.rs @@ -34,8 +34,6 @@ fn main() { } } - - fn tcp_listen() { use std::io::{Acceptor, Listener, TcpListener}; use std::io::util::copy; diff --git a/multipart/src/lib.rs b/multipart/src/lib.rs index 11cab64e7..6b0fff334 100644 --- a/multipart/src/lib.rs +++ b/multipart/src/lib.rs @@ -123,7 +123,7 @@ mod test { multipart.add_text("hello", "world"); multipart.add_text("goodnight", "sun"); - //multipart.sized = true; + multipart.sized = true; multipart.send(request).unwrap(); } From 9eb82ec101b8a560ad1e63879ef5322881ccaa00 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Mon, 24 Nov 2014 03:18:48 -0800 Subject: [PATCH 017/453] Add Travis-CI integration --- multipart/.travis.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 multipart/.travis.yml diff --git a/multipart/.travis.yml b/multipart/.travis.yml new file mode 100644 index 000000000..81ca6265f --- /dev/null +++ b/multipart/.travis.yml @@ -0,0 +1,21 @@ +language: rust +os: +- linux +- osx +env: + global: + - secure: eJpqXoY/xuCQRYmLBK9dLLzTGHvsBWe67wRN0fljesRgSnU+H09OsWI22DB9z/e6pMKjZpfDuyRkJaHVj8N7rwEfSeKdywv6uKqs5/Mi7dVINnzVA7jU3E4kLc+EuASF8w5d85kfccGNGs7qM0Uwz/i4da3Gw4B+cSc5cqjWsIY= +before_script: +- rustc -v +- cargo -V +script: +- cargo build -v +- ./target/multipart_server& +- sleep 5s #Wait for previous script to bind +- cargo test -v +- cargo doc -v +- kill $! #kill ./target/multippart_server +after_success: +- cp -R target/doc doc +- curl http://www.rust-ci.org/artifacts/put?t=$RUSTCI_TOKEN | sh +- rm -r doc From 343939fa306faa213e3b6bbb48b80f19d96bd474 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Mon, 24 Nov 2014 03:27:56 -0800 Subject: [PATCH 018/453] Don't build docs for dependencies --- multipart/.travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/.travis.yml b/multipart/.travis.yml index 81ca6265f..78996fa8c 100644 --- a/multipart/.travis.yml +++ b/multipart/.travis.yml @@ -13,7 +13,7 @@ script: - ./target/multipart_server& - sleep 5s #Wait for previous script to bind - cargo test -v -- cargo doc -v +- cargo doc -v --no-deps - kill $! #kill ./target/multippart_server after_success: - cp -R target/doc doc From cb5478164233b29242ee37f7c1c24a0821847efe Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Mon, 24 Nov 2014 03:44:14 -0800 Subject: [PATCH 019/453] Update README.md --- multipart/README.md | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/multipart/README.md b/multipart/README.md index 71cffb3d5..178d43bc5 100644 --- a/multipart/README.md +++ b/multipart/README.md @@ -1,4 +1,18 @@ -multipart +Multipart + Hyper [![Build Status](https://travis-ci.org/cybergeek94/multipart.svg?branch=master)](https://travis-ci.org/cybergeek94/multipart) ========= -An extension to hyperium/hyper that adds support for HTTP multipart requests +An extension to [Hyper][1] that adds support for HTTP multipart (`Content-Type: multipart/form-data`) requests. + +See `src/bin/multipart_server.rs` for server-side example (used in testing). + +####[Documentation][2] + +#####TODO: +- [ ] Fill out README and provide examples +- [ ] Improve documentation +- [ ] Add support for multiple files per field (nested boundaries) +- [ ] Expand test coverage +- [ ] Integrate with future HTTP compression extension(s) + +[1]: https://github.com/hyperium/hyper +[2]: http://rust-ci.org/cybergeek94/multipart/doc/multipart/ From 452b3b9c5b5bdaadadfb0c0f32dcfdd62c137a36 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Mon, 24 Nov 2014 03:49:43 -0800 Subject: [PATCH 020/453] Update README.md --- multipart/README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/multipart/README.md b/multipart/README.md index 178d43bc5..be403b083 100644 --- a/multipart/README.md +++ b/multipart/README.md @@ -7,7 +7,12 @@ See `src/bin/multipart_server.rs` for server-side example (used in testing). ####[Documentation][2] + +#####Warning +Using this library with `RUST_LOG=debug` will **flood** your logs due to excess `debug!()` usage during development. This will be fixed in a future update. I suggest you apply more specific log filtering, such as `RUST_LOG==debug`. + #####TODO: +- [ ] Remove excess debug statements - [ ] Fill out README and provide examples - [ ] Improve documentation - [ ] Add support for multiple files per field (nested boundaries) From b4e5d8cd9858503140a29d34f4f78e2c685523cb Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Mon, 24 Nov 2014 10:18:08 -0800 Subject: [PATCH 021/453] Remove excess debug statements --- multipart/src/server.rs | 34 ++-------------------------------- 1 file changed, 2 insertions(+), 32 deletions(-) diff --git a/multipart/src/server.rs b/multipart/src/server.rs index dab929542..07b4d9a05 100644 --- a/multipart/src/server.rs +++ b/multipart/src/server.rs @@ -254,21 +254,13 @@ impl BoundaryReader where S: Reader { } fn read_to_boundary(&mut self) -> IoResult<()> { - debug!("Read to boundary!"); - if !self.boundary_read { try!(self.true_fill_buf()); - debug!("Exited true_fill_buf"); - if self.buf_len == 0 { return eof(); } let lookahead = self.buf[self.last_search_idx .. self.buf_len]; - - debug!("Buf len: {}", self.buf_len); - - debug!("Lookahead: {}", lookahead.to_ascii().as_str_ascii()); - + let search_idx = lookahead.position_elem(&self.boundary[0]) .unwrap_or(lookahead.len() - 1); @@ -277,8 +269,6 @@ impl BoundaryReader where S: Reader { self.boundary_read = lookahead[search_idx..] .starts_with(self.boundary[]); - debug!("Boundary read: {} Boundary: {}", self.boundary_read, self.boundary.to_ascii().as_str_ascii()); - self.last_search_idx += search_idx; if !self.boundary_read { @@ -294,33 +284,23 @@ impl BoundaryReader where S: Reader { /// Read bytes until the reader is full fn true_fill_buf(&mut self) -> IoResult<()> { - debug!("True fill buf! Buf len: {}", self.buf_len); - let mut bytes_read = 1u; while bytes_read != 0 { - debug!("Bytes read loop!"); - bytes_read = match self.reader.read(self.buf[mut self.buf_len..]) { Ok(read) => read, Err(err) => if err.kind == EndOfFile { break; } else { return Err(err); }, }; - debug!("Bytes read: {}", bytes_read); - self.buf_len += bytes_read; } - debug!("Exited bytes read loop!"); - Ok(()) } fn _consume(&mut self, amt: uint) { use std::ptr::copy_memory; - - debug!("Consume! Amt: {}", amt); - + assert!(amt <= self.buf_len); let src = self.buf[amt..].as_ptr(); @@ -333,11 +313,7 @@ impl BoundaryReader where S: Reader { } fn consume_boundary(&mut self) -> IoResult<()> { - debug!("Consume boundary!"); - while !self.boundary_read { - debug!("Boundary read loop!"); - match self.read_to_boundary() { Ok(_) => (), Err(e) => if e.kind == EndOfFile { @@ -370,8 +346,6 @@ impl Reader for BoundaryReader where S: Reader { use std::cmp; use std::slice::bytes::copy_memory; - debug!("Read!"); - try!(self.read_to_boundary()); let trunc_len = cmp::min(buf.len(), self.last_search_idx); @@ -385,13 +359,9 @@ impl Reader for BoundaryReader where S: Reader { impl Buffer for BoundaryReader where S: Reader { fn fill_buf<'a>(&'a mut self) -> IoResult<&'a [u8]> { - debug!("Fill buf!"); - try!(self.read_to_boundary()); let buf = self.buf[..self.last_search_idx]; - - debug!("Buf: {}", buf.to_ascii().as_str_ascii()); Ok(buf) } From e8499fcf5eb8a00f3e8b8f2cd8f7efe17fe66aec Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Mon, 24 Nov 2014 10:19:11 -0800 Subject: [PATCH 022/453] Update README.md --- multipart/README.md | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/multipart/README.md b/multipart/README.md index be403b083..3dd316448 100644 --- a/multipart/README.md +++ b/multipart/README.md @@ -7,12 +7,8 @@ See `src/bin/multipart_server.rs` for server-side example (used in testing). ####[Documentation][2] - -#####Warning -Using this library with `RUST_LOG=debug` will **flood** your logs due to excess `debug!()` usage during development. This will be fixed in a future update. I suggest you apply more specific log filtering, such as `RUST_LOG==debug`. - #####TODO: -- [ ] Remove excess debug statements +- [x] Remove excess debug statements - [ ] Fill out README and provide examples - [ ] Improve documentation - [ ] Add support for multiple files per field (nested boundaries) From 5f1408af380d5bc908830ccdb320622a90dbb3e8 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Mon, 24 Nov 2014 11:38:06 -0800 Subject: [PATCH 023/453] More debug removal --- multipart/src/server.rs | 29 +++-------------------------- 1 file changed, 3 insertions(+), 26 deletions(-) diff --git a/multipart/src/server.rs b/multipart/src/server.rs index 07b4d9a05..86d0a912c 100644 --- a/multipart/src/server.rs +++ b/multipart/src/server.rs @@ -56,8 +56,6 @@ impl<'a> Multipart<'a> { pub fn from_request(req: Request<'a>) -> Result, Request<'a>> { if !is_multipart_formdata(&req) { return Err(req); } - debug!("Is multipart!"); - let boundary = if let Some(boundary) = req.headers.get::() .and_then(get_boundary) { boundary } else { return Err(req); }; @@ -66,9 +64,7 @@ impl<'a> Multipart<'a> { Ok(Multipart { source: BoundaryReader::from_reader(req, format!("--{}\r\n", boundary)) }) } - pub fn read_entry<'a>(&'a mut self) -> IoResult<(String, MultipartField<'a>)> { - debug!("Read entry!"); - + pub fn read_entry(&'a mut self) -> IoResult<(String, MultipartField<'a>)> { try!(self.source.consume_boundary()); let (disp_type, field_name, filename) = try!(self.read_content_disposition()); @@ -103,8 +99,6 @@ impl<'a> Multipart<'a> { /// See https://www.reddit.com/r/rust/comments/2lkk4i/concrete_lifetime_vs_bound_lifetime/ pub fn foreach_entry FnMut(String, MultipartField<'a>)>(&mut self, mut f: F) { loop { - debug!("Loop!"); - match self.read_entry() { Ok((name, field)) => f(name, field), Err(err) => { @@ -119,29 +113,20 @@ impl<'a> Multipart<'a> { } fn read_content_disposition(&mut self) -> IoResult<(String, String, Option)> { - debug!("Read content disposition!"); let line = try!(self.source.read_line()); - debug!("Line: {}", line); - // Find the end of CONT_DISP in the line let disp_type = { const CONT_DISP: &'static str = "Content-Disposition:"; let disp_idx = try_find!(line[], find_str, CONT_DISP, "Content-Disposition subheader not found!", line) + CONT_DISP.len(); - - debug!("Disp idx: {} Line len: {}", disp_idx, line.len()); let disp_type_end = try_find!(line[disp_idx..], find, ';', "Error parsing Content-Disposition value!", line); - debug!("Disp end: {}", disp_type_end); - line[disp_idx .. disp_idx + disp_type_end].trim().into_string() }; - - debug!("Disp-type: {}", disp_type); let field_name = { const NAME: &'static str = "name=\""; @@ -149,18 +134,12 @@ impl<'a> Multipart<'a> { let name_idx = try_find!(line[], find_str, NAME, "Error parsing field name!", line) + NAME.len(); - debug!("Name idx: {}", name_idx); - let name_end = try_find!(line[name_idx ..], find, '"', "Error parsing field name!", line); - debug!("Name end: {}", name_end); - line[name_idx .. name_idx + name_end].into_string() // No trim here since it's in quotes. }; - debug!("Field name: {}", field_name); - let filename = { const FILENAME: &'static str = "filename=\""; @@ -169,9 +148,7 @@ impl<'a> Multipart<'a> { filename_idxs.map(|(start, end)| line[start .. start + end].into_string()) }; - - debug!("Filename: {}", filename); - + Ok((disp_type, field_name, filename)) } @@ -206,7 +183,7 @@ fn line_error(msg: &'static str, line: String) -> IoError { } /* FIXME: Can't have an iterator return a borrowed reference -impl<'a> Iterator<(String, MultipartField<'a>)> for Multipart { +impl<'a> Iterator<(String, MultipartField<'a>)> for Multipart<'a> { fn next(&mut self) -> Option<(String, MultipartField<'a>)> { match self.read_entry() { Ok(ok) => Some(ok), From 4080cdad4e240ce0f47ba2db1d87f9114f69e531 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Mon, 24 Nov 2014 13:07:17 -0800 Subject: [PATCH 024/453] Implement file saving --- multipart/src/lib.rs | 77 +++++++++++++++++++++++++++++++++-------- multipart/src/server.rs | 55 +++++++++++++++++++++++++++-- 2 files changed, 116 insertions(+), 16 deletions(-) diff --git a/multipart/src/lib.rs b/multipart/src/lib.rs index 6b0fff334..d8d292ba0 100644 --- a/multipart/src/lib.rs +++ b/multipart/src/lib.rs @@ -10,7 +10,7 @@ use self::mime::Mime; use std::fmt::{Formatter, Show}; use std::fmt::Error as FormatError; -use std::io::{File, IoErrorKind, IoResult}; +use std::io::{File, IoErrorKind, IoResult, TempDir}; use std::io::fs::PathExtensions; @@ -18,18 +18,34 @@ pub mod client; pub mod server; pub mod mime_guess; +/// A representation of a file in HTTP `multipart/form-data`. +/// +/// This struct has an input "flavor" and an output "flavor". +/// The input "flavor" is used internally by `client::Multipart::add_file()` +/// and is never exposed to the user. +/// +/// The output "flavor" is returned by `server::Multipart::read_entry()` and represents +/// a file entry in the incoming multipart request. +/// +/// Note that in the output "flavor", the file is not yet saved to the system; +/// instead, the struct implements a `Reader` that points +/// to the beginning of the file's contents in the HTTP stream. +/// You can read it to EOF, or use one of the `save_*()` methods here +/// to save it to disk. pub struct MultipartFile<'a> { filename: Option, content_type: Mime, reader: &'a mut Reader + 'a, + tmp_dir: Option<&'a str>, } impl<'a> MultipartFile<'a> { - fn from_octet(filename: Option, reader: &'a mut Reader, cont_type: &str) -> MultipartFile<'a> { + fn from_octet(filename: Option, reader: &'a mut Reader, cont_type: &str, tmp_dir: &'a str) -> MultipartFile<'a> { MultipartFile { filename: filename, reader: reader, content_type: from_str(cont_type).unwrap_or_else(mime_guess::octet_stream), + tmp_dir: Some(tmp_dir), } } @@ -38,34 +54,67 @@ impl<'a> MultipartFile<'a> { filename: filename, reader: reader, content_type: mime, + tmp_dir: None, } } - /// Save this file to `path`, ignoring the filename, if any. + /// Save this file to `path`, discarding the filename. /// - /// Returns the created file on success. - pub fn save_as(&mut self, path: &Path) -> IoResult { + /// If successful, the file can be found at `path`. + pub fn save_as(&mut self, path: &Path) -> IoResult<()> { let mut file = try!(File::create(path)); - try!(ref_copy(self.reader, &mut file)); - - Ok(file) + ref_copy(self.reader, &mut file) } /// Save this file in the directory described by `dir`, - /// appending `filename` if any, or a random string. + /// appending `filename` if present, or a random string otherwise. /// - /// Returns the created file on success. + /// Returns the created file's path on success. /// /// ###Panics /// If `dir` does not represent a directory. - pub fn save_in(&mut self, dir: &Path) -> IoResult { + pub fn save_in(&mut self, dir: &Path) -> IoResult { assert!(dir.is_dir(), "Given path is not a directory!"); - let filename = self.filename.as_ref().map_or_else(|| random_alphanumeric(10), |s| s.clone()); - let path = dir.join(filename); + let path = dir.join(self.dest_filename()); - self.save_as(&path) + try!(self.save_as(&path)); + + Ok(path) + } + + /// Save this file in the temp directory `tmpdir` if supplied, + /// or a random subdirectory under `std::os::tmp_dir()` otherwise. + /// The same directory is used for all files in the same request). + /// + /// + /// Returns the created file's path on success. + pub fn save_temp(&mut self, tmp_dir: Option<&TempDir>) -> IoResult { + use std::os; + + let dir = match tmp_dir { + Some(tmp_dir) => tmp_dir.path().clone(), + None => os::tmpdir().join(self.tmp_dir.unwrap()), + }; + + self.save_in(&dir) + } + + fn dest_filename(&self) -> String { + self.filename.as_ref().map_or_else(|| random_alphanumeric(10), |s| s.clone()) + } + + pub fn filename(&self) -> Option<&str> { + self.filename.as_ref().map(|s| s[]) + } + + /// Get the content type of this file. + /// On the client, it is guessed by the file extension. + /// On the server, it is retrieved from the request or assumed to be + /// `application/octet-stream`. + pub fn content_type(&self) -> Mime { + self.content_type.clone() } } diff --git a/multipart/src/server.rs b/multipart/src/server.rs index 86d0a912c..c846166bb 100644 --- a/multipart/src/server.rs +++ b/multipart/src/server.rs @@ -7,6 +7,8 @@ use super::{MultipartField, MultipartFile}; use std::cmp; +use std::collections::HashMap; + use std::io::{IoError, IoResult, EndOfFile, standard_error, OtherIoError}; fn is_multipart_formdata(req: &Request) -> bool { @@ -41,6 +43,7 @@ fn get_boundary(ct: &ContentType) -> Option { pub struct Multipart<'a> { source: BoundaryReader>, + tmp_dir: String, } macro_rules! try_find( @@ -61,7 +64,7 @@ impl<'a> Multipart<'a> { debug!("Boundary: {}", boundary); - Ok(Multipart { source: BoundaryReader::from_reader(req, format!("--{}\r\n", boundary)) }) + Ok(Multipart { source: BoundaryReader::from_reader(req, format!("--{}\r\n", boundary)), tmp_dir: ::random_alphanumeric(10) }) } pub fn read_entry(&'a mut self) -> IoResult<(String, MultipartField<'a>)> { @@ -80,7 +83,7 @@ impl<'a> Multipart<'a> { let _ = try!(self.source.read_line()); // Consume empty line Ok((field_name, MultipartField::File( - MultipartFile::from_octet(filename, &mut self.source, content_type[]) + MultipartFile::from_octet(filename, &mut self.source, content_type[], self.tmp_dir[]) ) )) } else { @@ -96,6 +99,7 @@ impl<'a> Multipart<'a> { /// Call `f` for each entry in the multipart request. /// This is a substitute for `Multipart` implementing `Iterator`, /// since `Iterator::next()` can't use bound lifetimes. + /// /// See https://www.reddit.com/r/rust/comments/2lkk4i/concrete_lifetime_vs_bound_lifetime/ pub fn foreach_entry FnMut(String, MultipartField<'a>)>(&mut self, mut f: F) { loop { @@ -164,6 +168,35 @@ impl<'a> Multipart<'a> { // Does not expect boundary= Ok(type_idx.map(|start| line[start + CONTENT_TYPE.len()..].trim().into_string())) } + + /// Read the request fully, parsing all fields and saving all files to the given directory or a + /// temporary, and return the result. + /// + /// If `dir` is none, chooses a random subdirectory under `std::os::tmpdir()`. + pub fn save_all(mut self, dir: Option<&Path>) -> IoResult { + let dir = dir.map_or_else(|| ::std::os::tmpdir().join(self.tmp_dir[]), |path| path.clone()); + + let mut entries = Entries::with_path(dir); + + loop { + match self.read_entry() { + Ok((name, MultipartField::Text(text))) => { entries.fields.insert(name, text); }, + Ok((name, MultipartField::File(mut file))) => { + let path = try!(file.save_in(&entries.dir)); + entries.files.insert(name, path); + }, + Err(err) => { + if err.kind != EndOfFile { + error!("Error reading Multipart: {}", err); + } + + break; + }, + } + } + + Ok(entries) + } } fn with(left: Option, right: |&T| -> Option) -> Option<(T, U)> { @@ -182,6 +215,24 @@ fn line_error(msg: &'static str, line: String) -> IoError { } } +/// A result of `Multipart::save_all()`. +pub struct Entries { + pub fields: HashMap, + pub files: HashMap, + /// The directory the files were saved under. + pub dir: Path, +} + +impl Entries { + fn with_path(path: Path) -> Entries { + Entries { + fields: HashMap::new(), + files: HashMap::new(), + dir: path, + } + } +} + /* FIXME: Can't have an iterator return a borrowed reference impl<'a> Iterator<(String, MultipartField<'a>)> for Multipart<'a> { fn next(&mut self) -> Option<(String, MultipartField<'a>)> { From f628322522cd4052c8164ba32da40bbba33f4557 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Tue, 25 Nov 2014 12:57:38 -0800 Subject: [PATCH 025/453] Update mime types source --- multipart/mime_types.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/mime_types.json b/multipart/mime_types.json index 252b279da..1ad9b1254 100644 --- a/multipart/mime_types.json +++ b/multipart/mime_types.json @@ -1,5 +1,5 @@ { - "_source":"https://rawgithubusercontent.com/samuelneff/MimeTypeMap/master/src/MimeTypeMap.cs", + "_source":"https://github.com/samuelneff/MimeTypeMap/blob/master/src/MimeTypeMap.cs", "323": "text/h323", "3g2": "video/3gpp2", "3gp": "video/3gpp", From fc89240ecaa5319fea9c3a0a291a79f5380c0627 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Tue, 25 Nov 2014 15:50:48 -0800 Subject: [PATCH 026/453] Update mime_guess.rs --- multipart/src/mime_guess.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/multipart/src/mime_guess.rs b/multipart/src/mime_guess.rs index 9dfa7b400..abc47ff82 100644 --- a/multipart/src/mime_guess.rs +++ b/multipart/src/mime_guess.rs @@ -26,12 +26,12 @@ pub fn guess_mime_type_filename(filename: &str) -> Mime { local_data_key!(mime_types_cache: RefCell>) /// Get the MIME type associated with a file extension -// MIME Types are cached in a task-local heap pub fn get_mime_type(ext: &str) -> Mime { if ext.is_empty() { return octet_stream(); } let ext = ext.into_string(); + // MIME Types are cached in a task-local heap let cache = if let Some(cache) = mime_types_cache.get() { cache } else { mime_types_cache.replace(Some(RefCell::new(HashMap::new()))); @@ -59,6 +59,7 @@ fn find_mime_type(ext: &str) -> Mime { .unwrap_or_else(octet_stream) } +/// Get the `Mime` type for `application/octet-stream` (generic binary stream) pub fn octet_stream() -> Mime { Mime(TopLevel::Application, SubLevel::Ext("octet-stream".into_string()), Vec::new()) } From 311c92ede94a657905ec080a38302e9292962b92 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Tue, 25 Nov 2014 15:51:30 -0800 Subject: [PATCH 027/453] Update mime_guess.rs --- multipart/src/mime_guess.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/multipart/src/mime_guess.rs b/multipart/src/mime_guess.rs index abc47ff82..29410fb54 100644 --- a/multipart/src/mime_guess.rs +++ b/multipart/src/mime_guess.rs @@ -26,6 +26,8 @@ pub fn guess_mime_type_filename(filename: &str) -> Mime { local_data_key!(mime_types_cache: RefCell>) /// Get the MIME type associated with a file extension +/// If there is no association for the extension, +/// `application/octet-stream` is assumed. pub fn get_mime_type(ext: &str) -> Mime { if ext.is_empty() { return octet_stream(); } From dd9470dbf89b5784a3792e2798a1d802b1189db9 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Tue, 25 Nov 2014 15:53:54 -0800 Subject: [PATCH 028/453] Update mime_guess.rs --- multipart/src/mime_guess.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/multipart/src/mime_guess.rs b/multipart/src/mime_guess.rs index 29410fb54..352f5f610 100644 --- a/multipart/src/mime_guess.rs +++ b/multipart/src/mime_guess.rs @@ -17,6 +17,9 @@ pub fn guess_mime_type(path: &Path) -> Mime { get_mime_type(ext) } +/// Extract the extensioin of `filename` and guess its MIME type. +/// If there is no extension, or the extension has no known MIME association, +/// `applicaton/octet-stream` is assumed. pub fn guess_mime_type_filename(filename: &str) -> Mime { let path = Path::new(filename); From 05fde76741f0ac6dc5011acdbc90f7f8d30c822c Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Tue, 25 Nov 2014 15:54:34 -0800 Subject: [PATCH 029/453] Update mime_guess.rs --- multipart/src/mime_guess.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/multipart/src/mime_guess.rs b/multipart/src/mime_guess.rs index 352f5f610..79687edbe 100644 --- a/multipart/src/mime_guess.rs +++ b/multipart/src/mime_guess.rs @@ -29,8 +29,8 @@ pub fn guess_mime_type_filename(filename: &str) -> Mime { local_data_key!(mime_types_cache: RefCell>) /// Get the MIME type associated with a file extension -/// If there is no association for the extension, -/// `application/octet-stream` is assumed. +/// If there is no association for the extension, or `ext` is empty, +/// `application/octet-stream` is returned. pub fn get_mime_type(ext: &str) -> Mime { if ext.is_empty() { return octet_stream(); } From 689aebd25e717c3f03cac921e00d447797ab87d0 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Fri, 28 Nov 2014 15:27:09 -0800 Subject: [PATCH 030/453] Fix for TLD changes --- multipart/src/mime_guess.rs | 52 ++++++++++++++++--------------------- 1 file changed, 23 insertions(+), 29 deletions(-) diff --git a/multipart/src/mime_guess.rs b/multipart/src/mime_guess.rs index 79687edbe..c405f7d51 100644 --- a/multipart/src/mime_guess.rs +++ b/multipart/src/mime_guess.rs @@ -2,11 +2,8 @@ use mime::{Mime, TopLevel, SubLevel}; use serialize::json; -use std::cell::RefCell; - use std::collections::HashMap; - /// Guess the MIME type of the `Path` by its extension. /// /// **Guess** is the operative word here, as the contents of a file @@ -26,42 +23,39 @@ pub fn guess_mime_type_filename(filename: &str) -> Mime { guess_mime_type(&path) } -local_data_key!(mime_types_cache: RefCell>) +const MIME_TYPES: &'static str = include_str!("../mime_types.json"); + +// Lazily initialized task-local hashmap +// TODO: Make this global since it's read-only +thread_local!(static MIMES: HashMap = load_mime_types()) /// Get the MIME type associated with a file extension /// If there is no association for the extension, or `ext` is empty, /// `application/octet-stream` is returned. pub fn get_mime_type(ext: &str) -> Mime { if ext.is_empty() { return octet_stream(); } + + MIMES.with(|cache| cache.get(ext).cloned()).unwrap_or_else(octet_stream) +} - let ext = ext.into_string(); - - // MIME Types are cached in a task-local heap - let cache = if let Some(cache) = mime_types_cache.get() { cache } - else { - mime_types_cache.replace(Some(RefCell::new(HashMap::new()))); - mime_types_cache.get().unwrap() - }; - - if let Some(mime_type) = cache.borrow().get(&ext) { - return mime_type.clone(); - } - - let mime_type = find_mime_type(&*ext); - - cache.borrow_mut().insert(ext, mime_type.clone()); - - mime_type +/// Load the known mime types from the MIME_TYPES json +fn load_mime_types() -> HashMap { + let map = if let json::Object(map) = json::from_str(MIME_TYPES).unwrap() { map } + else { unreachable!("MIME types should be supplied as a map!"); }; + + map.into_iter().filter_map(to_mime_mapping).collect() } -const MIME_TYPES: &'static str = include_str!("../mime_types.json"); +fn to_mime_mapping(val: (String, json::Json)) -> Option<(String, Mime)> { + if let (st, json::String(mime)) = val { + if st.char_at(0) == '_' { return None; } -/// Load the MIME_TYPES as JSON and try to locate `ext` -fn find_mime_type(ext: &str) -> Mime { - json::from_str(MIME_TYPES).unwrap() - .find(ext).and_then(|j| j.as_string()) - .and_then(from_str::) - .unwrap_or_else(octet_stream) + if let Some(mime) = from_str::(&*mime) { + return Some((st, mime)) + } + } + + None } /// Get the `Mime` type for `application/octet-stream` (generic binary stream) From 68050163dfb7a2766043760038c46dd1997c1cc7 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Fri, 28 Nov 2014 15:34:29 -0800 Subject: [PATCH 031/453] Enable Crates.io integration --- multipart/Cargo.toml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 054cd172a..2f659778d 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -1,10 +1,16 @@ [package] name = "multipart" -version = "0.0.1" +version = "0.0.2" authors = ["Austin Bonander "] -[dependencies.hyper] -git = "https://github.com/hyperium/hyper" +description = "An extension to the Hyper HTTP library that provides support for POST multipart/form-data requests on for both client and server." +repository = "http://github.com/cybergeek94/multipart" +documentation = "http://rust-ci.org/cybergeek94/multipart/doc/multipart/" + +license = "MIT" + +[dependencies] +hyper = "*" From 92abb65ef6f3b86e0247976f7672532529a524fc Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Fri, 28 Nov 2014 16:01:24 -0800 Subject: [PATCH 032/453] Add keywords --- multipart/Cargo.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 2f659778d..f6d17b544 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -12,5 +12,7 @@ documentation = "http://rust-ci.org/cybergeek94/multipart/doc/multipart/" license = "MIT" +keywords = ["multipart", "form", "data", "hyper", "http", "post", "upload"] + [dependencies] hyper = "*" From 345e8c10252a435047ba37a485f1bd72a944d048 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Fri, 28 Nov 2014 16:02:17 -0800 Subject: [PATCH 033/453] Remove extra keywords --- multipart/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index f6d17b544..56b9c9683 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -12,7 +12,7 @@ documentation = "http://rust-ci.org/cybergeek94/multipart/doc/multipart/" license = "MIT" -keywords = ["multipart", "form", "data", "hyper", "http", "post", "upload"] +keywords = ["form-data", "hyper", "http", "post", "upload"] [dependencies] hyper = "*" From 8eb7fc28fc24a09ba5520b77a6c2213001df3775 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Fri, 28 Nov 2014 16:04:29 -0800 Subject: [PATCH 034/453] Increment semver --- multipart/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 56b9c9683..229d450b3 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "multipart" -version = "0.0.2" +version = "0.0.3" authors = ["Austin Bonander "] description = "An extension to the Hyper HTTP library that provides support for POST multipart/form-data requests on for both client and server." From f1e462e5b8f7feb3fc818cbc6cb1fecd7bb7de6d Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Fri, 28 Nov 2014 16:26:48 -0800 Subject: [PATCH 035/453] Move multipart_server as test --- multipart/src/bin/multipart_server.rs | 55 ------------------------ multipart/src/lib.rs | 19 --------- multipart/tests/multipart_server.rs | 61 +++++++++++++++++++++++++++ 3 files changed, 61 insertions(+), 74 deletions(-) delete mode 100644 multipart/src/bin/multipart_server.rs create mode 100644 multipart/tests/multipart_server.rs diff --git a/multipart/src/bin/multipart_server.rs b/multipart/src/bin/multipart_server.rs deleted file mode 100644 index 59d9de022..000000000 --- a/multipart/src/bin/multipart_server.rs +++ /dev/null @@ -1,55 +0,0 @@ -#![feature(unboxed_closures, if_let, slicing_syntax)] -#![allow(dead_code)] - -extern crate hyper; -extern crate multipart; - -use self::hyper::server::{Server, Request, Response}; -use self::hyper::status; - -use self::multipart::server::Multipart; - -use std::io::net::ip::Ipv4Addr; - -fn hello(req: Request, mut res: Response) { - print_req(&req); - - let mut multipart = Multipart::from_request(req).ok().expect("Could not create multipart!"); - - multipart.foreach_entry(|&: name, content| println!("Name: {} Content: {}", name, content)); - - *res.status_mut() = status::Ok; - - res.start().unwrap().end().unwrap(); -} - -fn main() { - let args = std::os::args(); - - if args.iter().find(|s| "tcp"== s[]).is_some() { - tcp_listen(); - } else { - let server = Server::http(Ipv4Addr(127, 0, 0, 1), 1337); - server.listen(hello).unwrap(); - } -} - -fn tcp_listen() { - use std::io::{Acceptor, Listener, TcpListener}; - use std::io::util::copy; - - let tcp = TcpListener::bind(("localhost", 1337u16)).unwrap(); - - let ref mut stdout = std::io::stdout(); - - for conn in tcp.listen().unwrap().incoming() { - let ref mut conn = conn.unwrap(); - - copy(conn, stdout).unwrap(); - } -} - -fn print_req(req: &Request) { - println!("Request: \nRemote addr: {}\nMethod: {}\nHeaders: {}\nURI: {}", - req.remote_addr, req.method, req.headers, req.uri); -} diff --git a/multipart/src/lib.rs b/multipart/src/lib.rs index d8d292ba0..ff03c2069 100644 --- a/multipart/src/lib.rs +++ b/multipart/src/lib.rs @@ -158,23 +158,4 @@ fn random_alphanumeric(len: uint) -> String { task_rng().gen_ascii_chars().map(|ch| ch.to_lowercase()).take(len).collect() } -#[cfg(test)] -mod test { - use hyper::Url; - use hyper::client::request::Request as ClientReq; - use client::Multipart as ClientMulti; - #[test] - fn client_api_test() { - let request = ClientReq::post(Url::parse("http://localhost:1337/").unwrap()).unwrap(); - - let mut multipart = ClientMulti::new(); - - multipart.add_text("hello", "world"); - multipart.add_text("goodnight", "sun"); - multipart.sized = true; - - multipart.send(request).unwrap(); - } - -} diff --git a/multipart/tests/multipart_server.rs b/multipart/tests/multipart_server.rs new file mode 100644 index 000000000..37f980d55 --- /dev/null +++ b/multipart/tests/multipart_server.rs @@ -0,0 +1,61 @@ +#![feature(unboxed_closures, if_let, slicing_syntax)] +#![allow(dead_code)] + +extern crate hyper; +extern crate multipart; + +use self::hyper::server::{Listening, Server, Request, Response}; +use self::hyper::client::Request as ClientReq; +use self::hyper::{status, Url}; + +use self::multipart::server::Multipart; + +use self::multipart::client::Multipart as ClientMulti; + +use std::io::net::ip::Ipv4Addr; + +use std::rand::random; + +fn hello(req: Request, mut res: Response) { + print_req(&req); + + let mut multipart = Multipart::from_request(req).ok().expect("Could not create multipart!"); + + multipart.foreach_entry(|&: name, content| println!("Name: {} Content: {}", name, content)); + + *res.status_mut() = status::Ok; + + res.start().unwrap().end().unwrap(); +} + +thread_local!(static PORT: u16 = random()) + +fn server() -> Listening { + let server = PORT.with(|port| Server::http(Ipv4Addr(127, 0, 0, 1), *port)); + server.listen(hello).unwrap() +} + +fn print_req(req: &Request) { + println!("Request: \nRemote addr: {}\nMethod: {}\nHeaders: {}\nURI: {}", + req.remote_addr, req.method, req.headers, req.uri); +} + +#[test] +fn client_api_test() { + let mut server = server(); + + let address = PORT.with(|port| format!("http://localhost:{}/", port)); + + let request = ClientReq::post(Url::parse(&*address).unwrap()).unwrap(); + + let mut multipart = ClientMulti::new(); + + multipart.add_text("hello", "world"); + multipart.add_text("goodnight", "sun"); + multipart.sized = true; + + multipart.send(request).unwrap(); + + server.close().unwrap(); +} + From 991c5421a3d2e353bc5db877300eaf50d25faa07 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Fri, 28 Nov 2014 16:28:35 -0800 Subject: [PATCH 036/453] Remove echoes from server --- multipart/tests/multipart_server.rs | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/multipart/tests/multipart_server.rs b/multipart/tests/multipart_server.rs index 37f980d55..7cdb398ef 100644 --- a/multipart/tests/multipart_server.rs +++ b/multipart/tests/multipart_server.rs @@ -16,12 +16,10 @@ use std::io::net::ip::Ipv4Addr; use std::rand::random; -fn hello(req: Request, mut res: Response) { - print_req(&req); - +fn ok_serv(req: Request, mut res: Response) { let mut multipart = Multipart::from_request(req).ok().expect("Could not create multipart!"); - multipart.foreach_entry(|&: name, content| println!("Name: {} Content: {}", name, content)); + multipart.foreach_entry(|&: _, _| ()); *res.status_mut() = status::Ok; @@ -32,12 +30,7 @@ thread_local!(static PORT: u16 = random()) fn server() -> Listening { let server = PORT.with(|port| Server::http(Ipv4Addr(127, 0, 0, 1), *port)); - server.listen(hello).unwrap() -} - -fn print_req(req: &Request) { - println!("Request: \nRemote addr: {}\nMethod: {}\nHeaders: {}\nURI: {}", - req.remote_addr, req.method, req.headers, req.uri); + server.listen(ok_serv).unwrap() } #[test] From cafbfee1e9825100647b1025a0543344d816a034 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Fri, 28 Nov 2014 16:29:57 -0800 Subject: [PATCH 037/453] Remove multipart_server from Travis --- multipart/.travis.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/multipart/.travis.yml b/multipart/.travis.yml index 78996fa8c..68e9d699d 100644 --- a/multipart/.travis.yml +++ b/multipart/.travis.yml @@ -10,11 +10,8 @@ before_script: - cargo -V script: - cargo build -v -- ./target/multipart_server& -- sleep 5s #Wait for previous script to bind - cargo test -v - cargo doc -v --no-deps -- kill $! #kill ./target/multippart_server after_success: - cp -R target/doc doc - curl http://www.rust-ci.org/artifacts/put?t=$RUSTCI_TOKEN | sh From 70d4819de2bd8a42ca0fa0eac927b0b827f097a9 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Fri, 28 Nov 2014 16:35:46 -0800 Subject: [PATCH 038/453] Remove test code from lib.rs --- multipart/src/lib.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/multipart/src/lib.rs b/multipart/src/lib.rs index ff03c2069..36b26a53c 100644 --- a/multipart/src/lib.rs +++ b/multipart/src/lib.rs @@ -158,4 +158,3 @@ fn random_alphanumeric(len: uint) -> String { task_rng().gen_ascii_chars().map(|ch| ch.to_lowercase()).take(len).collect() } - From 70ee2168c0f6f335dd317601cc6af77b979926b0 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Fri, 28 Nov 2014 18:51:29 -0800 Subject: [PATCH 039/453] Massive documentation and ergonomics improvements --- multipart/src/client.rs | 80 +++++++++++++++++++++++++++++++++---- multipart/src/lib.rs | 49 +++++++++++++++++++++++ multipart/src/mime_guess.rs | 20 +++++++--- 3 files changed, 136 insertions(+), 13 deletions(-) diff --git a/multipart/src/client.rs b/multipart/src/client.rs index 3b66c08da..ad0faf4ec 100644 --- a/multipart/src/client.rs +++ b/multipart/src/client.rs @@ -1,3 +1,9 @@ +//! The client side implementation of `multipart/form-data` requests. +//! +//! Use this when sending POST requests with files to a server. +//! +//! See the `Multipart` struct for more info. + use hyper::client::{Request, Response}; use hyper::header::common::{ContentType, ContentLength}; @@ -7,7 +13,7 @@ use hyper::{HttpResult, HttpIoError}; use mime::{Mime, TopLevel, SubLevel, Attr, Value}; -use mime_guess::guess_mime_type; +use mime_guess::{guess_mime_type, octet_stream}; use std::io::IoResult; use std::io::fs::File; @@ -18,11 +24,17 @@ const BOUNDARY_LEN: uint = 8; type Fields<'a> = Vec<(String, MultipartField<'a>)>; + +/// The entry point of the client-side multipart API. +/// +/// Add text fields with `.add_text()` and files with `.add_file()`, +/// then obtain a `hyper::client::Request` object and pass it to `.send()`. pub struct Multipart<'a> { fields: Fields<'a>, boundary: String, /// If the request can be sized. /// If true, avoid using chunked requests. + /// Defaults to `false`. pub sized: bool, } @@ -31,6 +43,7 @@ type ReqWrite = Request; impl<'a> Multipart<'a> { + /// Create a new `Multipart` instance with an empty set of fields. pub fn new() -> Multipart<'a> { Multipart { fields: Vec::new(), @@ -39,22 +52,73 @@ impl<'a> Multipart<'a> { } } - pub fn add_text(&mut self, name: &str, val: &str) { - self.fields.push((name.into_string(), MultipartField::Text(val.into_string()))); + /// Add a text field to this multipart request. + /// `name` and `val` can be either owned `String` or `&str`. + /// Prefer `String` if you're trying to limit allocations and copies. + pub fn add_text(&mut self, name: N, val: V) { + self.add_field(name, MultipartField::Text(val.into_string())); } - /// Add the file to the multipart request, guessing its `Content-Type` from its extension - pub fn add_file(&mut self, name: &str, file: &'a mut File) { + /// Add the file to the multipart request, guessing its `Content-Type` + /// from its extension and supplying its filename. + /// + /// See `add_stream()`. + pub fn add_file(&mut self, name: N, file: &'a mut File) { let filename = file.path().filename_str().map(|s| s.into_string()); let content_type = guess_mime_type(file.path()); - self.fields.push((name.into_string(), - MultipartField::File(MultipartFile::from_file(filename, file, content_type)))); + self.add_field(name, + MultipartField::File(MultipartFile::from_file(filename, file, content_type)) + ); + } + + /// Add a `Reader` as a file field, supplying `filename` if given, + /// and `content_type` if given or `application/octet-stream` if not. + /// + /// ##Warning + /// The given `Reader` **must** be able to read to EOF (end of file/no more data). + /// If it never returns EOF it will be read to infinity (even if it reads 0 bytes forever) + /// and the request will never be completed. + /// + /// If `sized` is `true`, this adds an additional consequence of out-of-control + /// memory usage, as `Multipart` tries to read an infinite amount of data into memory. + /// + /// Use `std::io::util::LimitReader` if you wish to send data from a `Reader` + /// that will never return EOF otherwise. + pub fn add_stream(&mut self, name: N, reader: &'a mut Reader + 'a, + filename: Option, content_type: Option) { + self.add_field(name, + MultipartField::File(MultipartFile { + filename: filename, + content_type: content_type.unwrap_or_else(octet_stream), + reader: reader, + tmp_dir: None, + }) + ); + } + + fn add_field(&mut self, name: N, val: MultipartField<'a>) { + self.fields.push((name.into_string(), val)); } - /// Apply the appropriate headers to the `Request` and send the data. + /// Apply the appropriate headers to the `Request` (obtained from Hyper) and send the data. /// If `self.sized == true`, send a sized (non-chunked) request, setting the `Content-Length` /// header. Else, send a chunked request. + /// + /// Sized requests are more human-readable and use less bandwidth + /// (as chunking adds [significant visual noise and overhead][chunked-example]), + /// but they must be able to load their entirety, including the contents of all files + /// and streams, into memory so the request body can be measured and its size set + /// in the `Content-Length` header. + /// + /// Prefer chunked requests when sending very large or numerous files, + /// or when human-readability or bandwidth aren't an issue. + /// + /// [chunked-example]: http://en.wikipedia.org/wiki/Chunked_transfer_encoding#Example + /// + /// ##Panics + /// If `req` is not a POST request, created with `Request::post()` or passing + /// `hyper::method::Post` to `Request::new()`. pub fn send(self, mut req: Request) -> HttpResult { use hyper::method; assert!(req.method() == method::Post, "Multipart request must use POST method!"); diff --git a/multipart/src/lib.rs b/multipart/src/lib.rs index 36b26a53c..6b2a4354c 100644 --- a/multipart/src/lib.rs +++ b/multipart/src/lib.rs @@ -124,13 +124,62 @@ impl<'a> Show for MultipartFile<'a> { } } + +/// A field in a `multipart/form-data` request. +/// +/// Like `MultipartFile`, this is used in both the client and server-side implementations, +/// but only exposed to the user on the server. +/// +/// This enum does not include the names of the fields, as those are yielded separately +/// by `server::Multipart::read_entry()`. #[deriving(Show)] pub enum MultipartField<'a> { + /// A text field. Text(String), + /// A file field, including the content type and optional filename + /// along with a `Reader` implementation for getting the contents. File(MultipartFile<'a>), // MultiFiles(Vec), /* TODO: Multiple files */ } +impl<'a> MultipartField<'a> { + + /// Borrow this field as a text field, if possible. + pub fn as_text<'a>(&'a self) -> Option<&'a str> { + match *self { + MultipartField::Text(ref s) => Some(s[]), + _ => None, + } + } + + /// Take this field as a text field, if possible, + /// returning `self` otherwise. + pub fn to_text(self) -> Result> { + match self { + MultipartField::Text(s) => Ok(s), + _ => Err(self), + } + } + + /// Borrow this field as a file field, if possible + /// Mutably borrows so the contents can be read. + pub fn as_file<'b>(&'b mut self) -> Option<&'b mut MultipartFile<'a>> { + match *self { + MultipartField::File(ref mut file) => Some(file), + _ => None, + } + } + + /// Take this field as a file field if possible, + /// returning `self` otherwise. + pub fn to_file(self) -> Result, MultipartField<'a>> { + match self { + MultipartField::File(file) => Ok(file), + _ => Err(self), + } + } +} + impl<'a> Reader for MultipartFile<'a> { fn read(&mut self, buf: &mut [u8]) -> IoResult{ self.reader.read(buf) diff --git a/multipart/src/mime_guess.rs b/multipart/src/mime_guess.rs index c405f7d51..ff6c56aea 100644 --- a/multipart/src/mime_guess.rs +++ b/multipart/src/mime_guess.rs @@ -1,3 +1,11 @@ +//! Guessing of MIME types by file extension. +//! +//! Extension-to-MIME-type mappings are loaded from a JSON file at runtime +//! and stored in a thread-local heap. To limit memory usage, avoid calling +//! functions in this module from several threads. +//! +//! TODO: Make mapping cache global in a safe manner. + use mime::{Mime, TopLevel, SubLevel}; use serialize::json; @@ -7,14 +15,15 @@ use std::collections::HashMap; /// Guess the MIME type of the `Path` by its extension. /// /// **Guess** is the operative word here, as the contents of a file -/// may not or may not match its MIME type/extension. +/// may not necessarily match its MIME type/extension. pub fn guess_mime_type(path: &Path) -> Mime { let ext = path.extension_str().unwrap_or(""); get_mime_type(ext) } -/// Extract the extensioin of `filename` and guess its MIME type. +/// Extract the extension of `filename` and guess its MIME type. +/// /// If there is no extension, or the extension has no known MIME association, /// `applicaton/octet-stream` is assumed. pub fn guess_mime_type_filename(filename: &str) -> Mime { @@ -26,10 +35,11 @@ pub fn guess_mime_type_filename(filename: &str) -> Mime { const MIME_TYPES: &'static str = include_str!("../mime_types.json"); // Lazily initialized task-local hashmap -// TODO: Make this global since it's read-only +// TODO: Make this global since it's read-only after init thread_local!(static MIMES: HashMap = load_mime_types()) -/// Get the MIME type associated with a file extension +/// Get the MIME type associated with a file extension. +/// /// If there is no association for the extension, or `ext` is empty, /// `application/octet-stream` is returned. pub fn get_mime_type(ext: &str) -> Mime { @@ -58,7 +68,7 @@ fn to_mime_mapping(val: (String, json::Json)) -> Option<(String, Mime)> { None } -/// Get the `Mime` type for `application/octet-stream` (generic binary stream) +/// Get the MIME type for `application/octet-stream` (generic binary stream) pub fn octet_stream() -> Mime { Mime(TopLevel::Application, SubLevel::Ext("octet-stream".into_string()), Vec::new()) } From 8cb7c692ad39b35b746ae72d62db8306cd1d2917 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Fri, 28 Nov 2014 18:52:58 -0800 Subject: [PATCH 040/453] Move server to subfolder to accomodate submodules --- multipart/src/server/mod.rs | 470 ++++++++++++++++++++++++++++++++++++ 1 file changed, 470 insertions(+) create mode 100644 multipart/src/server/mod.rs diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs new file mode 100644 index 000000000..fefcb1a58 --- /dev/null +++ b/multipart/src/server/mod.rs @@ -0,0 +1,470 @@ +//! The server-side implementation of `multipart/form-data` requests. +//! +//! Use this when you are implementing a server on top of Hyper and want to +//! to parse and serve POST `multipart/form-data` requests. +//! +//! See the `Multipart` struct for more info. + +use hyper::header::common::content_type::ContentType; +use hyper::server::request::Request; + +use mime::{Mime, TopLevel, SubLevel, Attr, Value}; + +use super::{MultipartField, MultipartFile}; + +use std::cmp; + +use std::collections::HashMap; + +use std::io::{IoError, IoResult, EndOfFile, standard_error, OtherIoError}; + +pub use self::handler::*; + +mod handler; + +fn is_multipart_formdata(req: &Request) -> bool { + use mime::{Multipart}; + + req.headers.get::().map_or(false, |ct| { + let ContentType(ref mime) = *ct; + + debug!("Content-Type: {}", mime); + + match *mime { + Mime(TopLevel::Multipart, SubLevel::FormData, _) => true, + _ => false, + } + }) +} + +fn get_boundary(ct: &ContentType) -> Option { + let ContentType(ref mime) = *ct; + let Mime(_, _, ref params) = *mime; + + params.iter().find(|&&(ref name, _)| + if let Attr::Ext(ref name) = *name { + name[] == "boundary" + } else { false } + ).and_then(|&(_, ref val)| + if let Value::Ext(ref val) = *val { + Some(val.clone()) + } else { None } + ) +} + +/// A server-side `multipart/form-data` requests. +/// +/// Create this with `Multipart::from_request()` passing a `server::Request` object from Hyper, +/// then read individual entries with `.read_entry()` or process them all at once with +/// `.foreach_entry()`. +pub struct Multipart<'a> { + source: BoundaryReader>, + tmp_dir: String, +} + +macro_rules! try_find( + ($haystack:expr, $f:ident, $needle:expr, $err:expr, $line:expr) => ( + try!($haystack.$f($needle).ok_or(line_error($err, $line.clone()))) + ) +) + +impl<'a> Multipart<'a> { + + /// If the given `Request` is of `Content-Type: multipart/form-data`, return + /// the wrapped request as `Ok(Multipart)`, otherwise `Err(Request)`. + pub fn from_request(req: Request<'a>) -> Result, Request<'a>> { + if !is_multipart_formdata(&req) { return Err(req); } + + let boundary = if let Some(boundary) = req.headers.get::() + .and_then(get_boundary) { boundary } else { return Err(req); }; + + debug!("Boundary: {}", boundary); + + Ok(Multipart { source: BoundaryReader::from_reader(req, format!("--{}\r\n", boundary)), tmp_dir: ::random_alphanumeric(10) }) + } + + /// Read an entry from this multipart request, returning a pair with the field's name and + /// contents. This will return an End of File error if there are no more entries. + /// + /// To get to the data, you will need to match on `MultipartField`: + /// + /// ```rust + /// fn process_request(req: &mut Request, res: &mut Response) -> HttpResult<()> { + /// + /// } + /// ``` + /// + /// ##Warning + /// If the last returned entry had contents of type `MultipartField::File`, + /// calling this again will discard any unread contents of that entry! + pub fn read_entry(&'a mut self) -> IoResult<(String, MultipartField<'a>)> { + try!(self.source.consume_boundary()); + let (disp_type, field_name, filename) = try!(self.read_content_disposition()); + + if &*disp_type != "form-data" { + return Err(IoError { + kind: OtherIoError, + desc: "Content-Disposition value was not \"form-data\"", + detail: Some(format!("Content-Disposition: {}", disp_type)), + }); + } + + if let Some(content_type) = try!(self.read_content_type()) { + let _ = try!(self.source.read_line()); // Consume empty line + Ok((field_name, + MultipartField::File( + MultipartFile::from_octet(filename, &mut self.source, content_type[], self.tmp_dir[]) + ) + )) + } else { + // Empty line consumed by read_content_type() + let text = try!(self.source.read_to_string()); + // The last two characters are "\r\n". + // We can't do a simple trim because the content might be terminated + // with line separators we want to preserve. + Ok((field_name, MultipartField::Text(text[..text.len() - 2].into_string()))) + } + } + + /// Call `f` for each entry in the multipart request. + /// This is a substitute for `Multipart` implementing `Iterator`, + /// since `Iterator::next()` can't use bound lifetimes. + /// + /// See https://www.reddit.com/r/rust/comments/2lkk4i/concrete_lifetime_vs_bound_lifetime/ + pub fn foreach_entry FnMut(String, MultipartField<'a>)>(&mut self, mut f: F) { + loop { + match self.read_entry() { + Ok((name, field)) => f(name, field), + Err(err) => { + if err.kind != EndOfFile { + error!("Error reading Multipart: {}", err); + } + + break; + }, + } + } + } + + fn read_content_disposition(&mut self) -> IoResult<(String, String, Option)> { + let line = try!(self.source.read_line()); + + // Find the end of CONT_DISP in the line + let disp_type = { + const CONT_DISP: &'static str = "Content-Disposition:"; + + let disp_idx = try_find!(line[], find_str, CONT_DISP, + "Content-Disposition subheader not found!", line) + CONT_DISP.len(); + + let disp_type_end = try_find!(line[disp_idx..], find, ';', + "Error parsing Content-Disposition value!", line); + + line[disp_idx .. disp_idx + disp_type_end].trim().into_string() + }; + + let field_name = { + const NAME: &'static str = "name=\""; + + let name_idx = try_find!(line[], find_str, NAME, + "Error parsing field name!", line) + NAME.len(); + + let name_end = try_find!(line[name_idx ..], find, '"', + "Error parsing field name!", line); + + line[name_idx .. name_idx + name_end].into_string() // No trim here since it's in quotes. + }; + + let filename = { + const FILENAME: &'static str = "filename=\""; + + let filename_idx = line[].find_str(FILENAME).map(|idx| idx + FILENAME.len()); + let filename_idxs = with(filename_idx, |&start| line[start ..].find('"')); + + filename_idxs.map(|(start, end)| line[start .. start + end].into_string()) + }; + + Ok((disp_type, field_name, filename)) + } + + fn read_content_type(&mut self) -> IoResult> { + debug!("Read content type!"); + let line = try!(self.source.read_line()); + + const CONTENT_TYPE: &'static str = "Content-Type:"; + + let type_idx = (&*line).find_str(CONTENT_TYPE); + + // FIXME Will not properly parse for multiple files! + // Does not expect boundary= + Ok(type_idx.map(|start| line[start + CONTENT_TYPE.len()..].trim().into_string())) + } + + /// Read the request fully, parsing all fields and saving all files to the given directory or a + /// temporary, and return the result. + /// + /// If `dir` is none, chooses a random subdirectory under `std::os::tmpdir()`. + pub fn save_all(mut self, dir: Option<&Path>) -> IoResult { + let dir = dir.map_or_else(|| ::std::os::tmpdir().join(self.tmp_dir[]), |path| path.clone()); + + let mut entries = Entries::with_path(dir); + + loop { + match self.read_entry() { + Ok((name, MultipartField::Text(text))) => { entries.fields.insert(name, text); }, + Ok((name, MultipartField::File(mut file))) => { + let path = try!(file.save_in(&entries.dir)); + entries.files.insert(name, path); + }, + Err(err) => { + if err.kind != EndOfFile { + error!("Error reading Multipart: {}", err); + } + + break; + }, + } + } + + Ok(entries) + } +} + +fn with(left: Option, right: |&T| -> Option) -> Option<(T, U)> { + let temp = left.as_ref().and_then(right); + match (left, temp) { + (Some(lval), Some(rval)) => Some((lval, rval)), + _ => None, + } +} + +fn line_error(msg: &'static str, line: String) -> IoError { + IoError { + kind: OtherIoError, + desc: msg, + detail: Some(line), + } +} + +/// A result of `Multipart::save_all()`. +pub struct Entries { + pub fields: HashMap, + pub files: HashMap, + /// The directory the files were saved under. + pub dir: Path, +} + +impl Entries { + fn with_path(path: Path) -> Entries { + Entries { + fields: HashMap::new(), + files: HashMap::new(), + dir: path, + } + } +} + +/* FIXME: Can't have an iterator return a borrowed reference +impl<'a> Iterator<(String, MultipartField<'a>)> for Multipart<'a> { + fn next(&mut self) -> Option<(String, MultipartField<'a>)> { + match self.read_entry() { + Ok(ok) => Some(ok), + Err(err) => { + if err.kind != EndOfFile { + error!("Error reading Multipart: {}", err); + } + + None + }, + } + } +} +*/ + +/// A `Reader` that will yield bytes until it sees a given sequence. +pub struct BoundaryReader { + reader: S, + boundary: Vec, + last_search_idx: uint, + boundary_read: bool, + buf: Vec, + buf_len: uint, +} + +fn eof() -> IoResult { + Err(standard_error(EndOfFile)) +} + +const BUF_SIZE: uint = 1024 * 64; // 64k buffer + +impl BoundaryReader where S: Reader { + fn from_reader(reader: S, boundary: String) -> BoundaryReader { + let mut buf = Vec::with_capacity(BUF_SIZE); + unsafe { buf.set_len(BUF_SIZE); } + + BoundaryReader { + reader: reader, + boundary: boundary.into_bytes(), + last_search_idx: 0, + boundary_read: false, + buf: buf, + buf_len: 0, + } + } + + fn read_to_boundary(&mut self) -> IoResult<()> { + if !self.boundary_read { + try!(self.true_fill_buf()); + + if self.buf_len == 0 { return eof(); } + + let lookahead = self.buf[self.last_search_idx .. self.buf_len]; + + let search_idx = lookahead.position_elem(&self.boundary[0]) + .unwrap_or(lookahead.len() - 1); + + debug!("Search idx: {}", search_idx); + + self.boundary_read = lookahead[search_idx..] + .starts_with(self.boundary[]); + + self.last_search_idx += search_idx; + + if !self.boundary_read { + self.last_search_idx += 1; + } + + } else if self.last_search_idx == 0 { + return Err(standard_error(EndOfFile)) + } + + Ok(()) + } + + /// Read bytes until the reader is full + fn true_fill_buf(&mut self) -> IoResult<()> { + let mut bytes_read = 1u; + + while bytes_read != 0 { + bytes_read = match self.reader.read(self.buf[mut self.buf_len..]) { + Ok(read) => read, + Err(err) => if err.kind == EndOfFile { break; } else { return Err(err); }, + }; + + self.buf_len += bytes_read; + } + + Ok(()) + } + + fn _consume(&mut self, amt: uint) { + use std::ptr::copy_memory; + + assert!(amt <= self.buf_len); + + let src = self.buf[amt..].as_ptr(); + let dest = self.buf[mut].as_mut_ptr(); + + unsafe { copy_memory(dest, src, self.buf_len - amt); } + + self.buf_len -= amt; + self.last_search_idx -= amt; + } + + fn consume_boundary(&mut self) -> IoResult<()> { + while !self.boundary_read { + match self.read_to_boundary() { + Ok(_) => (), + Err(e) => if e.kind == EndOfFile { + break; + } else { + return Err(e); + } + } + } + + let consume_amt = cmp::min(self.buf_len, self.last_search_idx + self.boundary.len()); + + debug!("Consume amt: {} Buf len: {}", consume_amt, self.buf_len); + + self._consume(consume_amt); + self.last_search_idx = 0; + self.boundary_read = false; + + Ok(()) + } + + #[allow(unused)] + fn set_boundary(&mut self, boundary: String) { + self.boundary = boundary.into_bytes(); + } +} + +impl Reader for BoundaryReader where S: Reader { + fn read(&mut self, buf: &mut [u8]) -> IoResult { + use std::cmp; + use std::slice::bytes::copy_memory; + + try!(self.read_to_boundary()); + + let trunc_len = cmp::min(buf.len(), self.last_search_idx); + copy_memory(buf, self.buf[..trunc_len]); + + self._consume(trunc_len); + + Ok(trunc_len) + } +} + +impl Buffer for BoundaryReader where S: Reader { + fn fill_buf<'a>(&'a mut self) -> IoResult<&'a [u8]> { + try!(self.read_to_boundary()); + + let buf = self.buf[..self.last_search_idx]; + + Ok(buf) + } + + fn consume(&mut self, amt: uint) { + assert!(amt <= self.last_search_idx); + self._consume(amt); + } +} + +#[test] +fn test_boundary() { + use std::io::BufReader; + + const BOUNDARY: &'static str = "--boundary\r\n"; + const TEST_VAL: &'static str = "\r +--boundary\r +dashed-value-1\r +--boundary\r +dashed-value-2\r +--boundary\r +"; + + let test_reader = BufReader::new(TEST_VAL.as_bytes()); + let mut reader = BoundaryReader::from_reader(test_reader, BOUNDARY.into_string()); + + debug!("Read 1"); + let string = reader.read_to_string().unwrap(); + debug!("{}", string); + assert!(string[].trim().is_empty()); + + debug!("Consume 1"); + reader.consume_boundary().unwrap(); + + debug!("Read 2"); + assert_eq!(reader.read_to_string().unwrap()[].trim(), "dashed-value-1"); + + debug!("Consume 2"); + reader.consume_boundary().unwrap(); + + debug!("Read 3"); + assert_eq!(reader.read_to_string().unwrap()[].trim(), "dashed-value-2"); + + debug!("Consume 3"); + reader.consume_boundary().unwrap(); + +} + From 0aa41ed91f8342c8f97802facbb95beed200e1f0 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Fri, 28 Nov 2014 19:38:40 -0800 Subject: [PATCH 041/453] Move server to a subfolder, convenience and doc improvements --- multipart/src/server.rs | 440 -------------------------------- multipart/src/server/handler.rs | 75 ++++++ multipart/src/server/mod.rs | 37 ++- 3 files changed, 98 insertions(+), 454 deletions(-) delete mode 100644 multipart/src/server.rs create mode 100644 multipart/src/server/handler.rs diff --git a/multipart/src/server.rs b/multipart/src/server.rs deleted file mode 100644 index c846166bb..000000000 --- a/multipart/src/server.rs +++ /dev/null @@ -1,440 +0,0 @@ -use hyper::header::common::content_type::ContentType; -use hyper::server::request::Request; - -use mime::{Mime, TopLevel, SubLevel, Attr, Value}; - -use super::{MultipartField, MultipartFile}; - -use std::cmp; - -use std::collections::HashMap; - -use std::io::{IoError, IoResult, EndOfFile, standard_error, OtherIoError}; - -fn is_multipart_formdata(req: &Request) -> bool { - use mime::{Multipart}; - - req.headers.get::().map_or(false, |ct| { - let ContentType(ref mime) = *ct; - - debug!("Content-Type: {}", mime); - - match *mime { - Mime(TopLevel::Multipart, SubLevel::FormData, _) => true, - _ => false, - } - }) -} - -fn get_boundary(ct: &ContentType) -> Option { - let ContentType(ref mime) = *ct; - let Mime(_, _, ref params) = *mime; - - params.iter().find(|&&(ref name, _)| - if let Attr::Ext(ref name) = *name { - name[] == "boundary" - } else { false } - ).and_then(|&(_, ref val)| - if let Value::Ext(ref val) = *val { - Some(val.clone()) - } else { None } - ) -} - -pub struct Multipart<'a> { - source: BoundaryReader>, - tmp_dir: String, -} - -macro_rules! try_find( - ($haystack:expr, $f:ident, $needle:expr, $err:expr, $line:expr) => ( - try!($haystack.$f($needle).ok_or(line_error($err, $line.clone()))) - ) -) - -impl<'a> Multipart<'a> { - - /// If the given `Request` is of `Content-Type: multipart/form-data`, return - /// the wrapped request as `Ok(Multipart)`, otherwise `Err(Request)`. - pub fn from_request(req: Request<'a>) -> Result, Request<'a>> { - if !is_multipart_formdata(&req) { return Err(req); } - - let boundary = if let Some(boundary) = req.headers.get::() - .and_then(get_boundary) { boundary } else { return Err(req); }; - - debug!("Boundary: {}", boundary); - - Ok(Multipart { source: BoundaryReader::from_reader(req, format!("--{}\r\n", boundary)), tmp_dir: ::random_alphanumeric(10) }) - } - - pub fn read_entry(&'a mut self) -> IoResult<(String, MultipartField<'a>)> { - try!(self.source.consume_boundary()); - let (disp_type, field_name, filename) = try!(self.read_content_disposition()); - - if &*disp_type != "form-data" { - return Err(IoError { - kind: OtherIoError, - desc: "Content-Disposition value was not \"form-data\"", - detail: Some(format!("Content-Disposition: {}", disp_type)), - }); - } - - if let Some(content_type) = try!(self.read_content_type()) { - let _ = try!(self.source.read_line()); // Consume empty line - Ok((field_name, - MultipartField::File( - MultipartFile::from_octet(filename, &mut self.source, content_type[], self.tmp_dir[]) - ) - )) - } else { - // Empty line consumed by read_content_type() - let text = try!(self.source.read_to_string()); - // The last two characters are "\r\n". - // We can't do a simple trim because the content might be terminated - // with line separators we want to preserve. - Ok((field_name, MultipartField::Text(text[..text.len() - 2].into_string()))) - } - } - - /// Call `f` for each entry in the multipart request. - /// This is a substitute for `Multipart` implementing `Iterator`, - /// since `Iterator::next()` can't use bound lifetimes. - /// - /// See https://www.reddit.com/r/rust/comments/2lkk4i/concrete_lifetime_vs_bound_lifetime/ - pub fn foreach_entry FnMut(String, MultipartField<'a>)>(&mut self, mut f: F) { - loop { - match self.read_entry() { - Ok((name, field)) => f(name, field), - Err(err) => { - if err.kind != EndOfFile { - error!("Error reading Multipart: {}", err); - } - - break; - }, - } - } - } - - fn read_content_disposition(&mut self) -> IoResult<(String, String, Option)> { - let line = try!(self.source.read_line()); - - // Find the end of CONT_DISP in the line - let disp_type = { - const CONT_DISP: &'static str = "Content-Disposition:"; - - let disp_idx = try_find!(line[], find_str, CONT_DISP, - "Content-Disposition subheader not found!", line) + CONT_DISP.len(); - - let disp_type_end = try_find!(line[disp_idx..], find, ';', - "Error parsing Content-Disposition value!", line); - - line[disp_idx .. disp_idx + disp_type_end].trim().into_string() - }; - - let field_name = { - const NAME: &'static str = "name=\""; - - let name_idx = try_find!(line[], find_str, NAME, - "Error parsing field name!", line) + NAME.len(); - - let name_end = try_find!(line[name_idx ..], find, '"', - "Error parsing field name!", line); - - line[name_idx .. name_idx + name_end].into_string() // No trim here since it's in quotes. - }; - - let filename = { - const FILENAME: &'static str = "filename=\""; - - let filename_idx = line[].find_str(FILENAME).map(|idx| idx + FILENAME.len()); - let filename_idxs = with(filename_idx, |&start| line[start ..].find('"')); - - filename_idxs.map(|(start, end)| line[start .. start + end].into_string()) - }; - - Ok((disp_type, field_name, filename)) - } - - fn read_content_type(&mut self) -> IoResult> { - debug!("Read content type!"); - let line = try!(self.source.read_line()); - - const CONTENT_TYPE: &'static str = "Content-Type:"; - - let type_idx = (&*line).find_str(CONTENT_TYPE); - - // FIXME Will not properly parse for multiple files! - // Does not expect boundary= - Ok(type_idx.map(|start| line[start + CONTENT_TYPE.len()..].trim().into_string())) - } - - /// Read the request fully, parsing all fields and saving all files to the given directory or a - /// temporary, and return the result. - /// - /// If `dir` is none, chooses a random subdirectory under `std::os::tmpdir()`. - pub fn save_all(mut self, dir: Option<&Path>) -> IoResult { - let dir = dir.map_or_else(|| ::std::os::tmpdir().join(self.tmp_dir[]), |path| path.clone()); - - let mut entries = Entries::with_path(dir); - - loop { - match self.read_entry() { - Ok((name, MultipartField::Text(text))) => { entries.fields.insert(name, text); }, - Ok((name, MultipartField::File(mut file))) => { - let path = try!(file.save_in(&entries.dir)); - entries.files.insert(name, path); - }, - Err(err) => { - if err.kind != EndOfFile { - error!("Error reading Multipart: {}", err); - } - - break; - }, - } - } - - Ok(entries) - } -} - -fn with(left: Option, right: |&T| -> Option) -> Option<(T, U)> { - let temp = left.as_ref().and_then(right); - match (left, temp) { - (Some(lval), Some(rval)) => Some((lval, rval)), - _ => None, - } -} - -fn line_error(msg: &'static str, line: String) -> IoError { - IoError { - kind: OtherIoError, - desc: msg, - detail: Some(line), - } -} - -/// A result of `Multipart::save_all()`. -pub struct Entries { - pub fields: HashMap, - pub files: HashMap, - /// The directory the files were saved under. - pub dir: Path, -} - -impl Entries { - fn with_path(path: Path) -> Entries { - Entries { - fields: HashMap::new(), - files: HashMap::new(), - dir: path, - } - } -} - -/* FIXME: Can't have an iterator return a borrowed reference -impl<'a> Iterator<(String, MultipartField<'a>)> for Multipart<'a> { - fn next(&mut self) -> Option<(String, MultipartField<'a>)> { - match self.read_entry() { - Ok(ok) => Some(ok), - Err(err) => { - if err.kind != EndOfFile { - error!("Error reading Multipart: {}", err); - } - - None - }, - } - } -} -*/ - -/// A `Reader` that will yield bytes until it sees a given sequence. -pub struct BoundaryReader { - reader: S, - boundary: Vec, - last_search_idx: uint, - boundary_read: bool, - buf: Vec, - buf_len: uint, -} - -fn eof() -> IoResult { - Err(standard_error(EndOfFile)) -} - -const BUF_SIZE: uint = 1024 * 64; // 64k buffer - -impl BoundaryReader where S: Reader { - fn from_reader(reader: S, boundary: String) -> BoundaryReader { - let mut buf = Vec::with_capacity(BUF_SIZE); - unsafe { buf.set_len(BUF_SIZE); } - - BoundaryReader { - reader: reader, - boundary: boundary.into_bytes(), - last_search_idx: 0, - boundary_read: false, - buf: buf, - buf_len: 0, - } - } - - fn read_to_boundary(&mut self) -> IoResult<()> { - if !self.boundary_read { - try!(self.true_fill_buf()); - - if self.buf_len == 0 { return eof(); } - - let lookahead = self.buf[self.last_search_idx .. self.buf_len]; - - let search_idx = lookahead.position_elem(&self.boundary[0]) - .unwrap_or(lookahead.len() - 1); - - debug!("Search idx: {}", search_idx); - - self.boundary_read = lookahead[search_idx..] - .starts_with(self.boundary[]); - - self.last_search_idx += search_idx; - - if !self.boundary_read { - self.last_search_idx += 1; - } - - } else if self.last_search_idx == 0 { - return Err(standard_error(EndOfFile)) - } - - Ok(()) - } - - /// Read bytes until the reader is full - fn true_fill_buf(&mut self) -> IoResult<()> { - let mut bytes_read = 1u; - - while bytes_read != 0 { - bytes_read = match self.reader.read(self.buf[mut self.buf_len..]) { - Ok(read) => read, - Err(err) => if err.kind == EndOfFile { break; } else { return Err(err); }, - }; - - self.buf_len += bytes_read; - } - - Ok(()) - } - - fn _consume(&mut self, amt: uint) { - use std::ptr::copy_memory; - - assert!(amt <= self.buf_len); - - let src = self.buf[amt..].as_ptr(); - let dest = self.buf[mut].as_mut_ptr(); - - unsafe { copy_memory(dest, src, self.buf_len - amt); } - - self.buf_len -= amt; - self.last_search_idx -= amt; - } - - fn consume_boundary(&mut self) -> IoResult<()> { - while !self.boundary_read { - match self.read_to_boundary() { - Ok(_) => (), - Err(e) => if e.kind == EndOfFile { - break; - } else { - return Err(e); - } - } - } - - let consume_amt = cmp::min(self.buf_len, self.last_search_idx + self.boundary.len()); - - debug!("Consume amt: {} Buf len: {}", consume_amt, self.buf_len); - - self._consume(consume_amt); - self.last_search_idx = 0; - self.boundary_read = false; - - Ok(()) - } - - #[allow(unused)] - fn set_boundary(&mut self, boundary: String) { - self.boundary = boundary.into_bytes(); - } -} - -impl Reader for BoundaryReader where S: Reader { - fn read(&mut self, buf: &mut [u8]) -> IoResult { - use std::cmp; - use std::slice::bytes::copy_memory; - - try!(self.read_to_boundary()); - - let trunc_len = cmp::min(buf.len(), self.last_search_idx); - copy_memory(buf, self.buf[..trunc_len]); - - self._consume(trunc_len); - - Ok(trunc_len) - } -} - -impl Buffer for BoundaryReader where S: Reader { - fn fill_buf<'a>(&'a mut self) -> IoResult<&'a [u8]> { - try!(self.read_to_boundary()); - - let buf = self.buf[..self.last_search_idx]; - - Ok(buf) - } - - fn consume(&mut self, amt: uint) { - assert!(amt <= self.last_search_idx); - self._consume(amt); - } -} - -#[test] -fn test_boundary() { - use std::io::BufReader; - - const BOUNDARY: &'static str = "--boundary\r\n"; - const TEST_VAL: &'static str = "\r ---boundary\r -dashed-value-1\r ---boundary\r -dashed-value-2\r ---boundary\r -"; - - let test_reader = BufReader::new(TEST_VAL.as_bytes()); - let mut reader = BoundaryReader::from_reader(test_reader, BOUNDARY.into_string()); - - debug!("Read 1"); - let string = reader.read_to_string().unwrap(); - debug!("{}", string); - assert!(string[].trim().is_empty()); - - debug!("Consume 1"); - reader.consume_boundary().unwrap(); - - debug!("Read 2"); - assert_eq!(reader.read_to_string().unwrap()[].trim(), "dashed-value-1"); - - debug!("Consume 2"); - reader.consume_boundary().unwrap(); - - debug!("Read 3"); - assert_eq!(reader.read_to_string().unwrap()[].trim(), "dashed-value-2"); - - debug!("Consume 3"); - reader.consume_boundary().unwrap(); - -} - diff --git a/multipart/src/server/handler.rs b/multipart/src/server/handler.rs new file mode 100644 index 000000000..27702701b --- /dev/null +++ b/multipart/src/server/handler.rs @@ -0,0 +1,75 @@ +//! Convenient wrappers for `hyper::server::Handler` + +use hyper::server::{Handler, Request, Response}; + +use super::Multipart; + +/// A container that implements `hyper::server::Handler` which will switch +/// the handler implementation depending on if the incoming request is multipart or not. +/// +/// Create an instance with `new()` and pass it to `hyper::server::Server::listen()` where +/// you would normally pass a `Handler` instance, usually a static function. +/// +/// A convenient wrapper for `Multipart::from_request()`. +pub struct Switch { + normal: H, + multipart: M, +} + +impl Switch where H: Handler, M: MultipartHandler { + /// Create a new `Switch` instance where + /// `normal` handles normal Hyper requests and `multipart` handles Multipart requests + pub fn new(normal: H, multipart: M) -> Switch { + Switch { + normal: normal, + multipart: multipart, + } + } +} + +impl Handler for Switch where H: Handler, M: MultipartHandler { + fn handle(&self, req: Request, res: Response) { + match Multipart::from_request(req) { + Ok(multi) => self.multipart.handle_multipart(multi, res), + Err(req) => self.normal.handle(req, res), + } + } +} + +/// A trait defining a type that can handle an incoming multipart request. +/// +/// Extends to unboxed closures of the type `Fn(Multipart, Response)`, +/// and subsequently static functions. +/// +/// Since `Multipart` implements `Deref`, you can still access +/// the fields on `Request`, such as `Request::uri` or `Request::headers`. +pub trait MultipartHandler: Send + Sync { + /// Generate a response from this multipart request. + fn handle_multipart<'a>(&self, multipart: Multipart<'a>, response: Response); +} + +impl MultipartHandler for F where F: for<'a> Fn(Multipart<'a>, Response) + Send + Sync { + fn handle_multipart<'a>(&self, multipart: Multipart<'a>, response: Response) { + (*self)(multipart, response); + } +} + +/// A container for an unboxed closure that implements `hyper::server::Handler`. +/// +/// This exists because as of this writing, `Handler` is not automatically implemented for +/// compatible unboxed closures (though this will likely change). +/// +/// No private fields, instantiate directly. +pub struct UnboxedHandler { + /// The closure to call + pub f: F, +} + +impl Handler for UnboxedHandler where F: Fn(Request, Response) + Send + Sync { + fn handle(&self, req: Request, res: Response) { + (self.f)(req, res); + } +} + + + diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index fefcb1a58..92d8d0838 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -7,6 +7,7 @@ use hyper::header::common::content_type::ContentType; use hyper::server::request::Request; +use hyper::method::Method; use mime::{Mime, TopLevel, SubLevel, Attr, Value}; @@ -18,14 +19,12 @@ use std::collections::HashMap; use std::io::{IoError, IoResult, EndOfFile, standard_error, OtherIoError}; -pub use self::handler::*; - -mod handler; +pub mod handler; fn is_multipart_formdata(req: &Request) -> bool { use mime::{Multipart}; - req.headers.get::().map_or(false, |ct| { + req.method == Method::Post && req.headers.get::().map_or(false, |ct| { let ContentType(ref mime) = *ct; debug!("Content-Type: {}", mime); @@ -52,11 +51,14 @@ fn get_boundary(ct: &ContentType) -> Option { ) } -/// A server-side `multipart/form-data` requests. +/// The server-side implementation of `multipart/form-data` requests. /// /// Create this with `Multipart::from_request()` passing a `server::Request` object from Hyper, +/// or give Hyper a `handler::Switch` instance instead, /// then read individual entries with `.read_entry()` or process them all at once with /// `.foreach_entry()`. +/// +/// Implements `Deref` to allow access to read-only fields on `Request` without copying. pub struct Multipart<'a> { source: BoundaryReader>, tmp_dir: String, @@ -70,8 +72,11 @@ macro_rules! try_find( impl<'a> Multipart<'a> { - /// If the given `Request` is of `Content-Type: multipart/form-data`, return - /// the wrapped request as `Ok(Multipart)`, otherwise `Err(Request)`. + /// If the given `Request` is a POST request of `Content-Type: multipart/form-data`, + /// return the wrapped request as `Ok(Multipart)`, otherwise `Err(Request)`. + /// + /// See the `handler` submodule for a convenient wrapper for this function, + /// `Switch`, that implements `hyper::server::Handler`. pub fn from_request(req: Request<'a>) -> Result, Request<'a>> { if !is_multipart_formdata(&req) { return Err(req); } @@ -86,13 +91,7 @@ impl<'a> Multipart<'a> { /// Read an entry from this multipart request, returning a pair with the field's name and /// contents. This will return an End of File error if there are no more entries. /// - /// To get to the data, you will need to match on `MultipartField`: - /// - /// ```rust - /// fn process_request(req: &mut Request, res: &mut Response) -> HttpResult<()> { - /// - /// } - /// ``` + /// To get to the data, you will need to match on `MultipartField`. /// /// ##Warning /// If the last returned entry had contents of type `MultipartField::File`, @@ -229,6 +228,12 @@ impl<'a> Multipart<'a> { } } +impl<'a> Deref> for Multipart<'a> { + fn deref(&self) -> &Request<'a> { + self.source.borrow_reader() + } +} + fn with(left: Option, right: |&T| -> Option) -> Option<(T, U)> { let temp = left.as_ref().and_then(right); match (left, temp) { @@ -397,6 +402,10 @@ impl BoundaryReader where S: Reader { fn set_boundary(&mut self, boundary: String) { self.boundary = boundary.into_bytes(); } + + pub fn borrow_reader<'a>(&'a self) -> &'a S { + &self.reader + } } impl Reader for BoundaryReader where S: Reader { From fcb049202fdaead8c3830a54a5eec8a0317ea88c Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Fri, 28 Nov 2014 19:39:24 -0800 Subject: [PATCH 042/453] Increment semver (minor version) --- multipart/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 229d450b3..87e6eef81 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "multipart" -version = "0.0.3" +version = "0.1.0" authors = ["Austin Bonander "] description = "An extension to the Hyper HTTP library that provides support for POST multipart/form-data requests on for both client and server." From 0818a3955fafa63df2e5f9a071e9e2bd849b4959 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Thu, 4 Dec 2014 00:30:33 -0800 Subject: [PATCH 043/453] Fix compiler errors --- multipart/src/client.rs | 11 ++++++----- multipart/src/lib.rs | 2 +- multipart/src/mime_guess.rs | 6 +++--- multipart/src/server/mod.rs | 2 -- multipart/tests/multipart_server.rs | 6 ++++-- 5 files changed, 14 insertions(+), 13 deletions(-) diff --git a/multipart/src/client.rs b/multipart/src/client.rs index ad0faf4ec..2ee16a8db 100644 --- a/multipart/src/client.rs +++ b/multipart/src/client.rs @@ -8,8 +8,10 @@ use hyper::client::{Request, Response}; use hyper::header::common::{ContentType, ContentLength}; +use hyper::method::Method; + use hyper::net::{Fresh, Streaming}; -use hyper::{HttpResult, HttpIoError}; +use hyper::{HttpResult, HttpError}; use mime::{Mime, TopLevel, SubLevel, Attr, Value}; @@ -85,7 +87,7 @@ impl<'a> Multipart<'a> { /// /// Use `std::io::util::LimitReader` if you wish to send data from a `Reader` /// that will never return EOF otherwise. - pub fn add_stream(&mut self, name: N, reader: &'a mut Reader + 'a, + pub fn add_stream(&mut self, name: N, reader: &'a mut (Reader + 'a), filename: Option, content_type: Option) { self.add_field(name, MultipartField::File(MultipartFile { @@ -120,8 +122,7 @@ impl<'a> Multipart<'a> { /// If `req` is not a POST request, created with `Request::post()` or passing /// `hyper::method::Post` to `Request::new()`. pub fn send(self, mut req: Request) -> HttpResult { - use hyper::method; - assert!(req.method() == method::Post, "Multipart request must use POST method!"); + assert!(req.method() == Method::Post, "Multipart request must use POST method!"); debug!("Fields: {}; Boundary: {}", self.fields[], self.boundary[]); @@ -203,7 +204,7 @@ fn write_line(req: &mut Writer, s: &str) -> IoResult<()> { fn io_to_http(res: IoResult) -> HttpResult { - res.map_err(|e| HttpIoError(e)) + res.map_err(|e| HttpError::HttpIoError(e)) } fn multipart_mime(bound: &str) -> Mime { diff --git a/multipart/src/lib.rs b/multipart/src/lib.rs index 6b2a4354c..d028f95ac 100644 --- a/multipart/src/lib.rs +++ b/multipart/src/lib.rs @@ -35,7 +35,7 @@ pub mod mime_guess; pub struct MultipartFile<'a> { filename: Option, content_type: Mime, - reader: &'a mut Reader + 'a, + reader: &'a mut (Reader + 'a), tmp_dir: Option<&'a str>, } diff --git a/multipart/src/mime_guess.rs b/multipart/src/mime_guess.rs index ff6c56aea..c6954abb2 100644 --- a/multipart/src/mime_guess.rs +++ b/multipart/src/mime_guess.rs @@ -8,7 +8,7 @@ use mime::{Mime, TopLevel, SubLevel}; -use serialize::json; +use serialize::json::{mod, Json}; use std::collections::HashMap; @@ -50,14 +50,14 @@ pub fn get_mime_type(ext: &str) -> Mime { /// Load the known mime types from the MIME_TYPES json fn load_mime_types() -> HashMap { - let map = if let json::Object(map) = json::from_str(MIME_TYPES).unwrap() { map } + let map = if let Json::Object(map) = json::from_str(MIME_TYPES).unwrap() { map } else { unreachable!("MIME types should be supplied as a map!"); }; map.into_iter().filter_map(to_mime_mapping).collect() } fn to_mime_mapping(val: (String, json::Json)) -> Option<(String, Mime)> { - if let (st, json::String(mime)) = val { + if let (st, Json::String(mime)) = val { if st.char_at(0) == '_' { return None; } if let Some(mime) = from_str::(&*mime) { diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index 92d8d0838..455c1c08b 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -22,8 +22,6 @@ use std::io::{IoError, IoResult, EndOfFile, standard_error, OtherIoError}; pub mod handler; fn is_multipart_formdata(req: &Request) -> bool { - use mime::{Multipart}; - req.method == Method::Post && req.headers.get::().map_or(false, |ct| { let ContentType(ref mime) = *ct; diff --git a/multipart/tests/multipart_server.rs b/multipart/tests/multipart_server.rs index 7cdb398ef..67daafba4 100644 --- a/multipart/tests/multipart_server.rs +++ b/multipart/tests/multipart_server.rs @@ -6,7 +6,9 @@ extern crate multipart; use self::hyper::server::{Listening, Server, Request, Response}; use self::hyper::client::Request as ClientReq; -use self::hyper::{status, Url}; +use self::hyper::status::StatusCode; +use self::hyper::Url; + use self::multipart::server::Multipart; @@ -21,7 +23,7 @@ fn ok_serv(req: Request, mut res: Response) { multipart.foreach_entry(|&: _, _| ()); - *res.status_mut() = status::Ok; + *res.status_mut() = StatusCode::Ok; res.start().unwrap().end().unwrap(); } From 8dc2847839ec0185bb115fb7eb7fda1ecd80da1e Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Fri, 5 Dec 2014 11:49:58 -0800 Subject: [PATCH 044/453] Revert to git version of hyper --- multipart/Cargo.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 87e6eef81..ec7e18a0b 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "multipart" -version = "0.1.0" +version = "0.1.1" authors = ["Austin Bonander "] description = "An extension to the Hyper HTTP library that provides support for POST multipart/form-data requests on for both client and server." @@ -14,5 +14,5 @@ license = "MIT" keywords = ["form-data", "hyper", "http", "post", "upload"] -[dependencies] -hyper = "*" +[dependencies.hyper] +git = "https://github.com/hyperium/hyper" From 8a5708d83a641352e710e5513de3e7a748ab6ed0 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Fri, 5 Dec 2014 11:51:23 -0800 Subject: [PATCH 045/453] Remove superfluous code: `HttpError` implicitly converts to `IoError` in the `try!` macro. --- multipart/src/client.rs | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/multipart/src/client.rs b/multipart/src/client.rs index 2ee16a8db..72a72812c 100644 --- a/multipart/src/client.rs +++ b/multipart/src/client.rs @@ -11,7 +11,7 @@ use hyper::header::common::{ContentType, ContentLength}; use hyper::method::Method; use hyper::net::{Fresh, Streaming}; -use hyper::{HttpResult, HttpError}; +use hyper::HttpResult; use mime::{Mime, TopLevel, SubLevel, Attr, Value}; @@ -26,7 +26,6 @@ const BOUNDARY_LEN: uint = 8; type Fields<'a> = Vec<(String, MultipartField<'a>)>; - /// The entry point of the client-side multipart API. /// /// Add text fields with `.add_text()` and files with `.add_file()`, @@ -137,7 +136,7 @@ impl<'a> Multipart<'a> { debug!("{}", req.headers()); let mut req = try!(req.start()); - try!(io_to_http(write_body(&mut req, fields, boundary[]))); + try!(write_body(&mut req, fields, boundary[])); req.send() } @@ -151,7 +150,7 @@ impl<'a> Multipart<'a> { apply_headers(&mut req, boundary[], Some(body.len())); let mut req = try!(req.start()); - try!(io_to_http(req.write(body[]))); + try!(req.write(body[])); req.send() } } @@ -202,11 +201,6 @@ fn write_line(req: &mut Writer, s: &str) -> IoResult<()> { req.write_str(s).and_then(|_| req.write(b"\r\n")) } - -fn io_to_http(res: IoResult) -> HttpResult { - res.map_err(|e| HttpError::HttpIoError(e)) -} - fn multipart_mime(bound: &str) -> Mime { Mime( TopLevel::Multipart, SubLevel::Ext("form-data".into_string()), From 7dd6d5998686a4d09c26aeef33500939cfc7d58e Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Fri, 27 Feb 2015 13:05:44 -0800 Subject: [PATCH 046/453] Upgrade to latest Rust --- multipart/Cargo.toml | 6 + multipart/src/client.rs | 211 ---------------------------- multipart/src/lib.rs | 209 --------------------------- multipart/src/mime_guess.rs | 83 ----------- multipart/src/server/handler.rs | 17 +-- multipart/src/server/mod.rs | 207 ++++++++++++++------------- multipart/tests/multipart_server.rs | 28 ++-- 7 files changed, 128 insertions(+), 633 deletions(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index ec7e18a0b..25e68f2ad 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -14,5 +14,11 @@ license = "MIT" keywords = ["form-data", "hyper", "http", "post", "upload"] +[dependencies] +log = "*" +mime = "*" +rand = "*" +rustc-serialize = "*" + [dependencies.hyper] git = "https://github.com/hyperium/hyper" diff --git a/multipart/src/client.rs b/multipart/src/client.rs index 72a72812c..e69de29bb 100644 --- a/multipart/src/client.rs +++ b/multipart/src/client.rs @@ -1,211 +0,0 @@ -//! The client side implementation of `multipart/form-data` requests. -//! -//! Use this when sending POST requests with files to a server. -//! -//! See the `Multipart` struct for more info. - -use hyper::client::{Request, Response}; - -use hyper::header::common::{ContentType, ContentLength}; - -use hyper::method::Method; - -use hyper::net::{Fresh, Streaming}; -use hyper::HttpResult; - -use mime::{Mime, TopLevel, SubLevel, Attr, Value}; - -use mime_guess::{guess_mime_type, octet_stream}; - -use std::io::IoResult; -use std::io::fs::File; - -use super::{MultipartField, MultipartFile, ref_copy, random_alphanumeric}; - -const BOUNDARY_LEN: uint = 8; - -type Fields<'a> = Vec<(String, MultipartField<'a>)>; - -/// The entry point of the client-side multipart API. -/// -/// Add text fields with `.add_text()` and files with `.add_file()`, -/// then obtain a `hyper::client::Request` object and pass it to `.send()`. -pub struct Multipart<'a> { - fields: Fields<'a>, - boundary: String, - /// If the request can be sized. - /// If true, avoid using chunked requests. - /// Defaults to `false`. - pub sized: bool, -} - -/// Shorthand for a writable request (`Request`) -type ReqWrite = Request; - -impl<'a> Multipart<'a> { - - /// Create a new `Multipart` instance with an empty set of fields. - pub fn new() -> Multipart<'a> { - Multipart { - fields: Vec::new(), - boundary: random_alphanumeric(BOUNDARY_LEN), - sized: false, - } - } - - /// Add a text field to this multipart request. - /// `name` and `val` can be either owned `String` or `&str`. - /// Prefer `String` if you're trying to limit allocations and copies. - pub fn add_text(&mut self, name: N, val: V) { - self.add_field(name, MultipartField::Text(val.into_string())); - } - - /// Add the file to the multipart request, guessing its `Content-Type` - /// from its extension and supplying its filename. - /// - /// See `add_stream()`. - pub fn add_file(&mut self, name: N, file: &'a mut File) { - let filename = file.path().filename_str().map(|s| s.into_string()); - let content_type = guess_mime_type(file.path()); - - self.add_field(name, - MultipartField::File(MultipartFile::from_file(filename, file, content_type)) - ); - } - - /// Add a `Reader` as a file field, supplying `filename` if given, - /// and `content_type` if given or `application/octet-stream` if not. - /// - /// ##Warning - /// The given `Reader` **must** be able to read to EOF (end of file/no more data). - /// If it never returns EOF it will be read to infinity (even if it reads 0 bytes forever) - /// and the request will never be completed. - /// - /// If `sized` is `true`, this adds an additional consequence of out-of-control - /// memory usage, as `Multipart` tries to read an infinite amount of data into memory. - /// - /// Use `std::io::util::LimitReader` if you wish to send data from a `Reader` - /// that will never return EOF otherwise. - pub fn add_stream(&mut self, name: N, reader: &'a mut (Reader + 'a), - filename: Option, content_type: Option) { - self.add_field(name, - MultipartField::File(MultipartFile { - filename: filename, - content_type: content_type.unwrap_or_else(octet_stream), - reader: reader, - tmp_dir: None, - }) - ); - } - - fn add_field(&mut self, name: N, val: MultipartField<'a>) { - self.fields.push((name.into_string(), val)); - } - - /// Apply the appropriate headers to the `Request` (obtained from Hyper) and send the data. - /// If `self.sized == true`, send a sized (non-chunked) request, setting the `Content-Length` - /// header. Else, send a chunked request. - /// - /// Sized requests are more human-readable and use less bandwidth - /// (as chunking adds [significant visual noise and overhead][chunked-example]), - /// but they must be able to load their entirety, including the contents of all files - /// and streams, into memory so the request body can be measured and its size set - /// in the `Content-Length` header. - /// - /// Prefer chunked requests when sending very large or numerous files, - /// or when human-readability or bandwidth aren't an issue. - /// - /// [chunked-example]: http://en.wikipedia.org/wiki/Chunked_transfer_encoding#Example - /// - /// ##Panics - /// If `req` is not a POST request, created with `Request::post()` or passing - /// `hyper::method::Post` to `Request::new()`. - pub fn send(self, mut req: Request) -> HttpResult { - assert!(req.method() == Method::Post, "Multipart request must use POST method!"); - - debug!("Fields: {}; Boundary: {}", self.fields[], self.boundary[]); - - if self.sized { - return self.send_sized(req); - } - - let Multipart { fields, boundary, ..} = self; - - apply_headers(&mut req, boundary[], None); - - debug!("{}", req.headers()); - - let mut req = try!(req.start()); - try!(write_body(&mut req, fields, boundary[])); - req.send() - } - - fn send_sized(self, mut req: Request) -> HttpResult { - let mut body: Vec = Vec::new(); - - let Multipart { fields, boundary, ..} = self; - - try!(write_body(&mut body, fields, boundary[])); - - apply_headers(&mut req, boundary[], Some(body.len())); - - let mut req = try!(req.start()); - try!(req.write(body[])); - req.send() - } -} - -fn apply_headers(req: &mut Request, boundary: &str, size: Option){ - let headers = req.headers_mut(); - - headers.set(ContentType(multipart_mime(boundary))); - - if let Some(size) = size { - headers.set(ContentLength(size)); - } -} - -fn write_body<'a>(wrt: &mut Writer, fields: Fields<'a>, boundary: &str) -> IoResult<()> { - try!(write_boundary(wrt, boundary[])); - - for (name, field) in fields.into_iter() { - try!(write_field(wrt, name, field, boundary)); - } - - Ok(()) -} - -fn write_field(wrt: &mut Writer, name: String, field: MultipartField, boundary: &str) -> IoResult<()> { - try!(write!(wrt, "Content-Disposition: form-data; name=\"{}\"\r\n\r\n", name)); - - try!(match field { - MultipartField::Text(text) => write_line(wrt, &*text), - MultipartField::File(file) => write_file(wrt, file), - }); - - write_boundary(wrt, boundary[]) -} - -fn write_boundary(wrt: &mut Writer, boundary: &str) -> IoResult<()> { - write!(wrt, "--{}\r\n", boundary) -} - -fn write_file(wrt: &mut Writer, mut file: MultipartFile) -> IoResult<()> { - try!(file.filename.map(|filename| write!(wrt, "; filename=\"{}\"\r\n", filename)).unwrap_or(Ok(()))); - try!(write!(wrt, "Content-Type: {}\r\n\r\n", file.content_type)); - ref_copy(&mut file.reader, wrt) -} - -/// Specialized write_line that writes CRLF after a line as per W3C specs -fn write_line(req: &mut Writer, s: &str) -> IoResult<()> { - req.write_str(s).and_then(|_| req.write(b"\r\n")) -} - -fn multipart_mime(bound: &str) -> Mime { - Mime( - TopLevel::Multipart, SubLevel::Ext("form-data".into_string()), - vec![(Attr::Ext("boundary".into_string()), Value::Ext(bound.into_string()))] - ) -} - - diff --git a/multipart/src/lib.rs b/multipart/src/lib.rs index d028f95ac..e69de29bb 100644 --- a/multipart/src/lib.rs +++ b/multipart/src/lib.rs @@ -1,209 +0,0 @@ -#![feature(if_let, slicing_syntax, default_type_params, phase, unboxed_closures, macro_rules)] -extern crate hyper; -#[phase(plugin, link)] extern crate log; - -extern crate mime; -extern crate serialize; - -use self::mime::Mime; - -use std::fmt::{Formatter, Show}; -use std::fmt::Error as FormatError; - -use std::io::{File, IoErrorKind, IoResult, TempDir}; - -use std::io::fs::PathExtensions; - -pub mod client; -pub mod server; -pub mod mime_guess; - -/// A representation of a file in HTTP `multipart/form-data`. -/// -/// This struct has an input "flavor" and an output "flavor". -/// The input "flavor" is used internally by `client::Multipart::add_file()` -/// and is never exposed to the user. -/// -/// The output "flavor" is returned by `server::Multipart::read_entry()` and represents -/// a file entry in the incoming multipart request. -/// -/// Note that in the output "flavor", the file is not yet saved to the system; -/// instead, the struct implements a `Reader` that points -/// to the beginning of the file's contents in the HTTP stream. -/// You can read it to EOF, or use one of the `save_*()` methods here -/// to save it to disk. -pub struct MultipartFile<'a> { - filename: Option, - content_type: Mime, - reader: &'a mut (Reader + 'a), - tmp_dir: Option<&'a str>, -} - -impl<'a> MultipartFile<'a> { - fn from_octet(filename: Option, reader: &'a mut Reader, cont_type: &str, tmp_dir: &'a str) -> MultipartFile<'a> { - MultipartFile { - filename: filename, - reader: reader, - content_type: from_str(cont_type).unwrap_or_else(mime_guess::octet_stream), - tmp_dir: Some(tmp_dir), - } - } - - fn from_file(filename: Option, reader: &'a mut File, mime: Mime) -> MultipartFile<'a> { - MultipartFile { - filename: filename, - reader: reader, - content_type: mime, - tmp_dir: None, - } - } - - /// Save this file to `path`, discarding the filename. - /// - /// If successful, the file can be found at `path`. - pub fn save_as(&mut self, path: &Path) -> IoResult<()> { - let mut file = try!(File::create(path)); - - ref_copy(self.reader, &mut file) - } - - /// Save this file in the directory described by `dir`, - /// appending `filename` if present, or a random string otherwise. - /// - /// Returns the created file's path on success. - /// - /// ###Panics - /// If `dir` does not represent a directory. - pub fn save_in(&mut self, dir: &Path) -> IoResult { - assert!(dir.is_dir(), "Given path is not a directory!"); - - let path = dir.join(self.dest_filename()); - - try!(self.save_as(&path)); - - Ok(path) - } - - /// Save this file in the temp directory `tmpdir` if supplied, - /// or a random subdirectory under `std::os::tmp_dir()` otherwise. - /// The same directory is used for all files in the same request). - /// - /// - /// Returns the created file's path on success. - pub fn save_temp(&mut self, tmp_dir: Option<&TempDir>) -> IoResult { - use std::os; - - let dir = match tmp_dir { - Some(tmp_dir) => tmp_dir.path().clone(), - None => os::tmpdir().join(self.tmp_dir.unwrap()), - }; - - self.save_in(&dir) - } - - fn dest_filename(&self) -> String { - self.filename.as_ref().map_or_else(|| random_alphanumeric(10), |s| s.clone()) - } - - pub fn filename(&self) -> Option<&str> { - self.filename.as_ref().map(|s| s[]) - } - - /// Get the content type of this file. - /// On the client, it is guessed by the file extension. - /// On the server, it is retrieved from the request or assumed to be - /// `application/octet-stream`. - pub fn content_type(&self) -> Mime { - self.content_type.clone() - } -} - -impl<'a> Show for MultipartFile<'a> { - fn fmt(&self, fmt: &mut Formatter) -> Result<(), FormatError> { - write!(fmt, "Filename: {} Content-Type: {}", self.filename, self.content_type) - } -} - - -/// A field in a `multipart/form-data` request. -/// -/// Like `MultipartFile`, this is used in both the client and server-side implementations, -/// but only exposed to the user on the server. -/// -/// This enum does not include the names of the fields, as those are yielded separately -/// by `server::Multipart::read_entry()`. -#[deriving(Show)] -pub enum MultipartField<'a> { - /// A text field. - Text(String), - /// A file field, including the content type and optional filename - /// along with a `Reader` implementation for getting the contents. - File(MultipartFile<'a>), - // MultiFiles(Vec), /* TODO: Multiple files */ -} - -impl<'a> MultipartField<'a> { - - /// Borrow this field as a text field, if possible. - pub fn as_text<'a>(&'a self) -> Option<&'a str> { - match *self { - MultipartField::Text(ref s) => Some(s[]), - _ => None, - } - } - - /// Take this field as a text field, if possible, - /// returning `self` otherwise. - pub fn to_text(self) -> Result> { - match self { - MultipartField::Text(s) => Ok(s), - _ => Err(self), - } - } - - /// Borrow this field as a file field, if possible - /// Mutably borrows so the contents can be read. - pub fn as_file<'b>(&'b mut self) -> Option<&'b mut MultipartFile<'a>> { - match *self { - MultipartField::File(ref mut file) => Some(file), - _ => None, - } - } - - /// Take this field as a file field if possible, - /// returning `self` otherwise. - pub fn to_file(self) -> Result, MultipartField<'a>> { - match self { - MultipartField::File(file) => Ok(file), - _ => Err(self), - } - } -} - -impl<'a> Reader for MultipartFile<'a> { - fn read(&mut self, buf: &mut [u8]) -> IoResult{ - self.reader.read(buf) - } -} - -/// A copy of `std::io::util::copy` that takes trait references -pub fn ref_copy(r: &mut Reader, w: &mut Writer) -> IoResult<()> { - let mut buf = [0, ..1024 * 64]; - - loop { - let len = match r.read(&mut buf) { - Ok(len) => len, - Err(ref e) if e.kind == IoErrorKind::EndOfFile => return Ok(()), - Err(e) => return Err(e), - }; - try!(w.write(buf[..len])); - } -} - -/// Generate a random alphanumeric sequence of length `len` -fn random_alphanumeric(len: uint) -> String { - use std::rand::{task_rng, Rng}; - - task_rng().gen_ascii_chars().map(|ch| ch.to_lowercase()).take(len).collect() -} - diff --git a/multipart/src/mime_guess.rs b/multipart/src/mime_guess.rs index c6954abb2..e69de29bb 100644 --- a/multipart/src/mime_guess.rs +++ b/multipart/src/mime_guess.rs @@ -1,83 +0,0 @@ -//! Guessing of MIME types by file extension. -//! -//! Extension-to-MIME-type mappings are loaded from a JSON file at runtime -//! and stored in a thread-local heap. To limit memory usage, avoid calling -//! functions in this module from several threads. -//! -//! TODO: Make mapping cache global in a safe manner. - -use mime::{Mime, TopLevel, SubLevel}; - -use serialize::json::{mod, Json}; - -use std::collections::HashMap; - -/// Guess the MIME type of the `Path` by its extension. -/// -/// **Guess** is the operative word here, as the contents of a file -/// may not necessarily match its MIME type/extension. -pub fn guess_mime_type(path: &Path) -> Mime { - let ext = path.extension_str().unwrap_or(""); - - get_mime_type(ext) -} - -/// Extract the extension of `filename` and guess its MIME type. -/// -/// If there is no extension, or the extension has no known MIME association, -/// `applicaton/octet-stream` is assumed. -pub fn guess_mime_type_filename(filename: &str) -> Mime { - let path = Path::new(filename); - - guess_mime_type(&path) -} - -const MIME_TYPES: &'static str = include_str!("../mime_types.json"); - -// Lazily initialized task-local hashmap -// TODO: Make this global since it's read-only after init -thread_local!(static MIMES: HashMap = load_mime_types()) - -/// Get the MIME type associated with a file extension. -/// -/// If there is no association for the extension, or `ext` is empty, -/// `application/octet-stream` is returned. -pub fn get_mime_type(ext: &str) -> Mime { - if ext.is_empty() { return octet_stream(); } - - MIMES.with(|cache| cache.get(ext).cloned()).unwrap_or_else(octet_stream) -} - -/// Load the known mime types from the MIME_TYPES json -fn load_mime_types() -> HashMap { - let map = if let Json::Object(map) = json::from_str(MIME_TYPES).unwrap() { map } - else { unreachable!("MIME types should be supplied as a map!"); }; - - map.into_iter().filter_map(to_mime_mapping).collect() -} - -fn to_mime_mapping(val: (String, json::Json)) -> Option<(String, Mime)> { - if let (st, Json::String(mime)) = val { - if st.char_at(0) == '_' { return None; } - - if let Some(mime) = from_str::(&*mime) { - return Some((st, mime)) - } - } - - None -} - -/// Get the MIME type for `application/octet-stream` (generic binary stream) -pub fn octet_stream() -> Mime { - Mime(TopLevel::Application, SubLevel::Ext("octet-stream".into_string()), Vec::new()) -} - -#[test] -fn test_mime_type_guessing() { - assert!(get_mime_type("gif").to_string() == "image/gif".to_string()); - assert!(get_mime_type("txt").to_string() == "text/plain".to_string()); - assert!(get_mime_type("blahblah").to_string() == "application/octet-stream".to_string()); -} - - diff --git a/multipart/src/server/handler.rs b/multipart/src/server/handler.rs index 27702701b..30434de9b 100644 --- a/multipart/src/server/handler.rs +++ b/multipart/src/server/handler.rs @@ -13,11 +13,11 @@ use super::Multipart; /// A convenient wrapper for `Multipart::from_request()`. pub struct Switch { normal: H, - multipart: M, + multipart: M, } impl Switch where H: Handler, M: MultipartHandler { - /// Create a new `Switch` instance where + /// Create a new `Switch` instance where /// `normal` handles normal Hyper requests and `multipart` handles Multipart requests pub fn new(normal: H, multipart: M) -> Switch { Switch { @@ -28,7 +28,7 @@ impl Switch where H: Handler, M: MultipartHandler { } impl Handler for Switch where H: Handler, M: MultipartHandler { - fn handle(&self, req: Request, res: Response) { + fn handle<'a>(&'a self, req: Request<'a>, res: Response<'a>) { match Multipart::from_request(req) { Ok(multi) => self.multipart.handle_multipart(multi, res), Err(req) => self.normal.handle(req, res), @@ -38,7 +38,7 @@ impl Handler for Switch where H: Handler, M: MultipartHandler { /// A trait defining a type that can handle an incoming multipart request. /// -/// Extends to unboxed closures of the type `Fn(Multipart, Response)`, +/// Extends to unboxed closures of the type `Fn(Multipart, Response)`, /// and subsequently static functions. /// /// Since `Multipart` implements `Deref`, you can still access @@ -50,13 +50,13 @@ pub trait MultipartHandler: Send + Sync { impl MultipartHandler for F where F: for<'a> Fn(Multipart<'a>, Response) + Send + Sync { fn handle_multipart<'a>(&self, multipart: Multipart<'a>, response: Response) { - (*self)(multipart, response); + (*self)(multipart, response); } } /// A container for an unboxed closure that implements `hyper::server::Handler`. /// -/// This exists because as of this writing, `Handler` is not automatically implemented for +/// This exists because as of this writing, `Handler` is not automatically implemented for /// compatible unboxed closures (though this will likely change). /// /// No private fields, instantiate directly. @@ -68,8 +68,5 @@ pub struct UnboxedHandler { impl Handler for UnboxedHandler where F: Fn(Request, Response) + Send + Sync { fn handle(&self, req: Request, res: Response) { (self.f)(req, res); - } + } } - - - diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index 455c1c08b..f0259054b 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -1,35 +1,34 @@ //! The server-side implementation of `multipart/form-data` requests. //! -//! Use this when you are implementing a server on top of Hyper and want to +//! Use this when you are implementing a server on top of Hyper and want to //! to parse and serve POST `multipart/form-data` requests. -//! +//! //! See the `Multipart` struct for more info. -use hyper::header::common::content_type::ContentType; +use hyper::header::ContentType; use hyper::server::request::Request; use hyper::method::Method; use mime::{Mime, TopLevel, SubLevel, Attr, Value}; -use super::{MultipartField, MultipartFile}; +use super::{IntoString, MultipartField, MultipartFile}; use std::cmp; - use std::collections::HashMap; - -use std::io::{IoError, IoResult, EndOfFile, standard_error, OtherIoError}; +use std::old_io::{IoError, IoResult, EndOfFile, standard_error, OtherIoError}; +use std::ops::Deref; pub mod handler; fn is_multipart_formdata(req: &Request) -> bool { req.method == Method::Post && req.headers.get::().map_or(false, |ct| { let ContentType(ref mime) = *ct; - + debug!("Content-Type: {}", mime); match *mime { Mime(TopLevel::Multipart, SubLevel::FormData, _) => true, - _ => false, + _ => false, } }) } @@ -37,16 +36,16 @@ fn is_multipart_formdata(req: &Request) -> bool { fn get_boundary(ct: &ContentType) -> Option { let ContentType(ref mime) = *ct; let Mime(_, _, ref params) = *mime; - - params.iter().find(|&&(ref name, _)| - if let Attr::Ext(ref name) = *name { - name[] == "boundary" + + params.iter().find(|&&(ref name, _)| + if let Attr::Ext(ref name) = *name { + "boundary" == &**name } else { false } - ).and_then(|&(_, ref val)| - if let Value::Ext(ref val) = *val { - Some(val.clone()) + ).and_then(|&(_, ref val)| + if let Value::Ext(ref val) = *val { + Some(val.clone()) } else { None } - ) + ) } /// The server-side implementation of `multipart/form-data` requests. @@ -66,11 +65,11 @@ macro_rules! try_find( ($haystack:expr, $f:ident, $needle:expr, $err:expr, $line:expr) => ( try!($haystack.$f($needle).ok_or(line_error($err, $line.clone()))) ) -) +); impl<'a> Multipart<'a> { - /// If the given `Request` is a POST request of `Content-Type: multipart/form-data`, + /// If the given `Request` is a POST request of `Content-Type: multipart/form-data`, /// return the wrapped request as `Ok(Multipart)`, otherwise `Err(Request)`. /// /// See the `handler` submodule for a convenient wrapper for this function, @@ -93,8 +92,8 @@ impl<'a> Multipart<'a> { /// /// ##Warning /// If the last returned entry had contents of type `MultipartField::File`, - /// calling this again will discard any unread contents of that entry! - pub fn read_entry(&'a mut self) -> IoResult<(String, MultipartField<'a>)> { + /// calling this again will discard any unread contents of that entry! + pub fn read_entry<'b>(&'b mut self) -> IoResult<(String, MultipartField<'b>)> { try!(self.source.consume_boundary()); let (disp_type, field_name, filename) = try!(self.read_content_disposition()); @@ -105,12 +104,12 @@ impl<'a> Multipart<'a> { detail: Some(format!("Content-Disposition: {}", disp_type)), }); } - + if let Some(content_type) = try!(self.read_content_type()) { let _ = try!(self.source.read_line()); // Consume empty line - Ok((field_name, + Ok((field_name, MultipartField::File( - MultipartFile::from_octet(filename, &mut self.source, content_type[], self.tmp_dir[]) + MultipartFile::from_octet(filename, &mut self.source, &content_type, &self.tmp_dir) ) )) } else { @@ -119,50 +118,50 @@ impl<'a> Multipart<'a> { // The last two characters are "\r\n". // We can't do a simple trim because the content might be terminated // with line separators we want to preserve. - Ok((field_name, MultipartField::Text(text[..text.len() - 2].into_string()))) - } + Ok((field_name, MultipartField::Text(text[..text.len() - 2].into_string()))) + } } - + /// Call `f` for each entry in the multipart request. /// This is a substitute for `Multipart` implementing `Iterator`, /// since `Iterator::next()` can't use bound lifetimes. /// - /// See https://www.reddit.com/r/rust/comments/2lkk4i/concrete_lifetime_vs_bound_lifetime/ - pub fn foreach_entry FnMut(String, MultipartField<'a>)>(&mut self, mut f: F) { + /// See https://www.reddit.com/r/rust/comments/2lkk\4\isize/concrete_lifetime_vs_bound_lifetime/ + pub fn foreach_entry FnMut(String, MultipartField<'b>)>(&'a mut self, mut f: F) { loop { match self.read_entry() { Ok((name, field)) => f(name, field), - Err(err) => { + Err(err) => { if err.kind != EndOfFile { error!("Error reading Multipart: {}", err); } break; - }, - } - } - } - + }, + } + } + } + fn read_content_disposition(&mut self) -> IoResult<(String, String, Option)> { - let line = try!(self.source.read_line()); + let line = try!(self.source.read_line()); // Find the end of CONT_DISP in the line let disp_type = { const CONT_DISP: &'static str = "Content-Disposition:"; - let disp_idx = try_find!(line[], find_str, CONT_DISP, + let disp_idx = try_find!(&line, find_str, CONT_DISP, "Content-Disposition subheader not found!", line) + CONT_DISP.len(); - let disp_type_end = try_find!(line[disp_idx..], find, ';', + let disp_type_end = try_find!(line[disp_idx..], find, ';', "Error parsing Content-Disposition value!", line); line[disp_idx .. disp_idx + disp_type_end].trim().into_string() }; - + let field_name = { const NAME: &'static str = "name=\""; - let name_idx = try_find!(line[], find_str, NAME, + let name_idx = try_find!(&line, find_str, NAME, "Error parsing field name!", line) + NAME.len(); let name_end = try_find!(line[name_idx ..], find, '"', @@ -174,12 +173,12 @@ impl<'a> Multipart<'a> { let filename = { const FILENAME: &'static str = "filename=\""; - let filename_idx = line[].find_str(FILENAME).map(|idx| idx + FILENAME.len()); + let filename_idx = line.find_str(FILENAME).map(|idx| idx + FILENAME.len()); let filename_idxs = with(filename_idx, |&start| line[start ..].find('"')); - + filename_idxs.map(|(start, end)| line[start .. start + end].into_string()) }; - + Ok((disp_type, field_name, filename)) } @@ -188,10 +187,10 @@ impl<'a> Multipart<'a> { let line = try!(self.source.read_line()); const CONTENT_TYPE: &'static str = "Content-Type:"; - + let type_idx = (&*line).find_str(CONTENT_TYPE); - // FIXME Will not properly parse for multiple files! + // FIXME Will not properly parse for multiple files! // Does not expect boundary= Ok(type_idx.map(|start| line[start + CONTENT_TYPE.len()..].trim().into_string())) } @@ -201,7 +200,7 @@ impl<'a> Multipart<'a> { /// /// If `dir` is none, chooses a random subdirectory under `std::os::tmpdir()`. pub fn save_all(mut self, dir: Option<&Path>) -> IoResult { - let dir = dir.map_or_else(|| ::std::os::tmpdir().join(self.tmp_dir[]), |path| path.clone()); + let dir = dir.map_or_else(|| ::std::os::tmpdir().join(&self.tmp_dir), |path| path.clone()); let mut entries = Entries::with_path(dir); @@ -211,8 +210,8 @@ impl<'a> Multipart<'a> { Ok((name, MultipartField::File(mut file))) => { let path = try!(file.save_in(&entries.dir)); entries.files.insert(name, path); - }, - Err(err) => { + }, + Err(err) => { if err.kind != EndOfFile { error!("Error reading Multipart: {}", err); } @@ -226,23 +225,24 @@ impl<'a> Multipart<'a> { } } -impl<'a> Deref> for Multipart<'a> { +impl<'a> Deref for Multipart<'a> { + type Target = Request<'a>; fn deref(&self) -> &Request<'a> { - self.source.borrow_reader() - } + self.source.borrow_reader() + } } -fn with(left: Option, right: |&T| -> Option) -> Option<(T, U)> { +fn with Option>(left: Option, right: F) -> Option<(T, U)> { let temp = left.as_ref().and_then(right); match (left, temp) { (Some(lval), Some(rval)) => Some((lval, rval)), - _ => None, + _ => None, } -} +} fn line_error(msg: &'static str, line: String) -> IoError { - IoError { - kind: OtherIoError, + IoError { + kind: OtherIoError, desc: msg, detail: Some(line), } @@ -263,15 +263,15 @@ impl Entries { files: HashMap::new(), dir: path, } - } + } } /* FIXME: Can't have an iterator return a borrowed reference impl<'a> Iterator<(String, MultipartField<'a>)> for Multipart<'a> { fn next(&mut self) -> Option<(String, MultipartField<'a>)> { match self.read_entry() { - Ok(ok) => Some(ok), - Err(err) => { + Ok(ok) => Some(ok), + Err(err) => { if err.kind != EndOfFile { error!("Error reading Multipart: {}", err); } @@ -279,7 +279,7 @@ impl<'a> Iterator<(String, MultipartField<'a>)> for Multipart<'a> { None }, } - } + } } */ @@ -287,17 +287,17 @@ impl<'a> Iterator<(String, MultipartField<'a>)> for Multipart<'a> { pub struct BoundaryReader { reader: S, boundary: Vec, - last_search_idx: uint, + last_search_idx: usize, boundary_read: bool, buf: Vec, - buf_len: uint, + buf_len: usize, } fn eof() -> IoResult { - Err(standard_error(EndOfFile)) + Err(standard_error(EndOfFile)) } -const BUF_SIZE: uint = 1024 * 64; // 64k buffer +const BUF_SIZE: usize = 1024 * 64; // 64k buffer impl BoundaryReader where S: Reader { fn from_reader(reader: S, boundary: String) -> BoundaryReader { @@ -310,7 +310,7 @@ impl BoundaryReader where S: Reader { last_search_idx: 0, boundary_read: false, buf: buf, - buf_len: 0, + buf_len: 0, } } @@ -319,36 +319,36 @@ impl BoundaryReader where S: Reader { try!(self.true_fill_buf()); if self.buf_len == 0 { return eof(); } - - let lookahead = self.buf[self.last_search_idx .. self.buf_len]; - + + let lookahead = &self.buf[self.last_search_idx .. self.buf_len]; + let search_idx = lookahead.position_elem(&self.boundary[0]) .unwrap_or(lookahead.len() - 1); debug!("Search idx: {}", search_idx); self.boundary_read = lookahead[search_idx..] - .starts_with(self.boundary[]); + .starts_with(&self.boundary); self.last_search_idx += search_idx; if !self.boundary_read { - self.last_search_idx += 1; + self.last_search_idx += 1; } } else if self.last_search_idx == 0 { - return Err(standard_error(EndOfFile)) + return Err(standard_error(EndOfFile)) } - - Ok(()) + + Ok(()) } /// Read bytes until the reader is full fn true_fill_buf(&mut self) -> IoResult<()> { - let mut bytes_read = 1u; - + let mut bytes_read = 1usize; + while bytes_read != 0 { - bytes_read = match self.reader.read(self.buf[mut self.buf_len..]) { + bytes_read = match self.reader.read(&mut self.buf[self.buf_len..]) { Ok(read) => read, Err(err) => if err.kind == EndOfFile { break; } else { return Err(err); }, }; @@ -359,46 +359,46 @@ impl BoundaryReader where S: Reader { Ok(()) } - fn _consume(&mut self, amt: uint) { + fn _consume(&mut self, amt: usize) { use std::ptr::copy_memory; - + assert!(amt <= self.buf_len); let src = self.buf[amt..].as_ptr(); - let dest = self.buf[mut].as_mut_ptr(); + let dest = self.buf.as_mut_ptr(); unsafe { copy_memory(dest, src, self.buf_len - amt); } - + self.buf_len -= amt; - self.last_search_idx -= amt; + self.last_search_idx -= amt; } fn consume_boundary(&mut self) -> IoResult<()> { while !self.boundary_read { match self.read_to_boundary() { Ok(_) => (), - Err(e) => if e.kind == EndOfFile { - break; - } else { + Err(e) => if e.kind == EndOfFile { + break; + } else { return Err(e); } } } - + let consume_amt = cmp::min(self.buf_len, self.last_search_idx + self.boundary.len()); debug!("Consume amt: {} Buf len: {}", consume_amt, self.buf_len); self._consume(consume_amt); self.last_search_idx = 0; - self.boundary_read = false; - - Ok(()) + self.boundary_read = false; + + Ok(()) } #[allow(unused)] fn set_boundary(&mut self, boundary: String) { - self.boundary = boundary.into_bytes(); + self.boundary = boundary.into_bytes(); } pub fn borrow_reader<'a>(&'a self) -> &'a S { @@ -407,31 +407,31 @@ impl BoundaryReader where S: Reader { } impl Reader for BoundaryReader where S: Reader { - fn read(&mut self, buf: &mut [u8]) -> IoResult { - use std::cmp; + fn read(&mut self, buf: &mut [u8]) -> IoResult { + use std::cmp; use std::slice::bytes::copy_memory; - try!(self.read_to_boundary()); + try!(self.read_to_boundary()); let trunc_len = cmp::min(buf.len(), self.last_search_idx); - copy_memory(buf, self.buf[..trunc_len]); + copy_memory(buf, &self.buf[..trunc_len]); self._consume(trunc_len); - Ok(trunc_len) - } + Ok(trunc_len) + } } impl Buffer for BoundaryReader where S: Reader { fn fill_buf<'a>(&'a mut self) -> IoResult<&'a [u8]> { try!(self.read_to_boundary()); - - let buf = self.buf[..self.last_search_idx]; - Ok(buf) + let buf = &self.buf[..self.last_search_idx]; + + Ok(buf) } - fn consume(&mut self, amt: uint) { + fn consume(&mut self, amt: usize) { assert!(amt <= self.last_search_idx); self._consume(amt); } @@ -456,22 +456,21 @@ dashed-value-2\r debug!("Read 1"); let string = reader.read_to_string().unwrap(); debug!("{}", string); - assert!(string[].trim().is_empty()); + assert!(string.trim().is_empty()); debug!("Consume 1"); reader.consume_boundary().unwrap(); debug!("Read 2"); - assert_eq!(reader.read_to_string().unwrap()[].trim(), "dashed-value-1"); + assert_eq!(reader.read_to_string().unwrap().trim(), "dashed-value-1"); debug!("Consume 2"); reader.consume_boundary().unwrap(); debug!("Read 3"); - assert_eq!(reader.read_to_string().unwrap()[].trim(), "dashed-value-2"); + assert_eq!(reader.read_to_string().unwrap().trim(), "dashed-value-2"); debug!("Consume 3"); reader.consume_boundary().unwrap(); - -} +} diff --git a/multipart/tests/multipart_server.rs b/multipart/tests/multipart_server.rs index 67daafba4..b808ced2c 100644 --- a/multipart/tests/multipart_server.rs +++ b/multipart/tests/multipart_server.rs @@ -1,34 +1,31 @@ -#![feature(unboxed_closures, if_let, slicing_syntax)] #![allow(dead_code)] extern crate hyper; extern crate multipart; -use self::hyper::server::{Listening, Server, Request, Response}; -use self::hyper::client::Request as ClientReq; -use self::hyper::status::StatusCode; -use self::hyper::Url; +use hyper::server::{Listening, Server, Request, Response}; +use hyper::client::Request as ClientReq; +use hyper::status::StatusCode; +use hyper::Url; +use multipart::server::Multipart; +use multipart::client::Multipart as ClientMulti; -use self::multipart::server::Multipart; - -use self::multipart::client::Multipart as ClientMulti; - -use std::io::net::ip::Ipv4Addr; +use std::old_io::net::ip::Ipv4Addr; use std::rand::random; -fn ok_serv(req: Request, mut res: Response) { +fn ok_serv(req: Request, mut res: Response) { let mut multipart = Multipart::from_request(req).ok().expect("Could not create multipart!"); multipart.foreach_entry(|&: _, _| ()); - + *res.status_mut() = StatusCode::Ok; res.start().unwrap().end().unwrap(); } -thread_local!(static PORT: u16 = random()) +thread_local!(static PORT: u16 = random()); fn server() -> Listening { let server = PORT.with(|port| Server::http(Ipv4Addr(127, 0, 0, 1), *port)); @@ -39,7 +36,7 @@ fn server() -> Listening { fn client_api_test() { let mut server = server(); - let address = PORT.with(|port| format!("http://localhost:{}/", port)); + let address = PORT.with(|port| format!("http://localhost:{}/", port)); let request = ClientReq::post(Url::parse(&*address).unwrap()).unwrap(); @@ -50,7 +47,6 @@ fn client_api_test() { multipart.sized = true; multipart.send(request).unwrap(); - + server.close().unwrap(); } - From f94f26c9945127d728cff238876287f43d874103 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sat, 4 Jul 2015 14:25:41 -0700 Subject: [PATCH 047/453] Revert "Upgrade to latest Rust" This reverts commit 7dd6d5998686a4d09c26aeef33500939cfc7d58e. --- multipart/Cargo.toml | 6 - multipart/src/client.rs | 211 ++++++++++++++++++++++++++++ multipart/src/lib.rs | 209 +++++++++++++++++++++++++++ multipart/src/mime_guess.rs | 83 +++++++++++ multipart/src/server/handler.rs | 17 ++- multipart/src/server/mod.rs | 207 +++++++++++++-------------- multipart/tests/multipart_server.rs | 28 ++-- 7 files changed, 633 insertions(+), 128 deletions(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 25e68f2ad..ec7e18a0b 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -14,11 +14,5 @@ license = "MIT" keywords = ["form-data", "hyper", "http", "post", "upload"] -[dependencies] -log = "*" -mime = "*" -rand = "*" -rustc-serialize = "*" - [dependencies.hyper] git = "https://github.com/hyperium/hyper" diff --git a/multipart/src/client.rs b/multipart/src/client.rs index e69de29bb..72a72812c 100644 --- a/multipart/src/client.rs +++ b/multipart/src/client.rs @@ -0,0 +1,211 @@ +//! The client side implementation of `multipart/form-data` requests. +//! +//! Use this when sending POST requests with files to a server. +//! +//! See the `Multipart` struct for more info. + +use hyper::client::{Request, Response}; + +use hyper::header::common::{ContentType, ContentLength}; + +use hyper::method::Method; + +use hyper::net::{Fresh, Streaming}; +use hyper::HttpResult; + +use mime::{Mime, TopLevel, SubLevel, Attr, Value}; + +use mime_guess::{guess_mime_type, octet_stream}; + +use std::io::IoResult; +use std::io::fs::File; + +use super::{MultipartField, MultipartFile, ref_copy, random_alphanumeric}; + +const BOUNDARY_LEN: uint = 8; + +type Fields<'a> = Vec<(String, MultipartField<'a>)>; + +/// The entry point of the client-side multipart API. +/// +/// Add text fields with `.add_text()` and files with `.add_file()`, +/// then obtain a `hyper::client::Request` object and pass it to `.send()`. +pub struct Multipart<'a> { + fields: Fields<'a>, + boundary: String, + /// If the request can be sized. + /// If true, avoid using chunked requests. + /// Defaults to `false`. + pub sized: bool, +} + +/// Shorthand for a writable request (`Request`) +type ReqWrite = Request; + +impl<'a> Multipart<'a> { + + /// Create a new `Multipart` instance with an empty set of fields. + pub fn new() -> Multipart<'a> { + Multipart { + fields: Vec::new(), + boundary: random_alphanumeric(BOUNDARY_LEN), + sized: false, + } + } + + /// Add a text field to this multipart request. + /// `name` and `val` can be either owned `String` or `&str`. + /// Prefer `String` if you're trying to limit allocations and copies. + pub fn add_text(&mut self, name: N, val: V) { + self.add_field(name, MultipartField::Text(val.into_string())); + } + + /// Add the file to the multipart request, guessing its `Content-Type` + /// from its extension and supplying its filename. + /// + /// See `add_stream()`. + pub fn add_file(&mut self, name: N, file: &'a mut File) { + let filename = file.path().filename_str().map(|s| s.into_string()); + let content_type = guess_mime_type(file.path()); + + self.add_field(name, + MultipartField::File(MultipartFile::from_file(filename, file, content_type)) + ); + } + + /// Add a `Reader` as a file field, supplying `filename` if given, + /// and `content_type` if given or `application/octet-stream` if not. + /// + /// ##Warning + /// The given `Reader` **must** be able to read to EOF (end of file/no more data). + /// If it never returns EOF it will be read to infinity (even if it reads 0 bytes forever) + /// and the request will never be completed. + /// + /// If `sized` is `true`, this adds an additional consequence of out-of-control + /// memory usage, as `Multipart` tries to read an infinite amount of data into memory. + /// + /// Use `std::io::util::LimitReader` if you wish to send data from a `Reader` + /// that will never return EOF otherwise. + pub fn add_stream(&mut self, name: N, reader: &'a mut (Reader + 'a), + filename: Option, content_type: Option) { + self.add_field(name, + MultipartField::File(MultipartFile { + filename: filename, + content_type: content_type.unwrap_or_else(octet_stream), + reader: reader, + tmp_dir: None, + }) + ); + } + + fn add_field(&mut self, name: N, val: MultipartField<'a>) { + self.fields.push((name.into_string(), val)); + } + + /// Apply the appropriate headers to the `Request` (obtained from Hyper) and send the data. + /// If `self.sized == true`, send a sized (non-chunked) request, setting the `Content-Length` + /// header. Else, send a chunked request. + /// + /// Sized requests are more human-readable and use less bandwidth + /// (as chunking adds [significant visual noise and overhead][chunked-example]), + /// but they must be able to load their entirety, including the contents of all files + /// and streams, into memory so the request body can be measured and its size set + /// in the `Content-Length` header. + /// + /// Prefer chunked requests when sending very large or numerous files, + /// or when human-readability or bandwidth aren't an issue. + /// + /// [chunked-example]: http://en.wikipedia.org/wiki/Chunked_transfer_encoding#Example + /// + /// ##Panics + /// If `req` is not a POST request, created with `Request::post()` or passing + /// `hyper::method::Post` to `Request::new()`. + pub fn send(self, mut req: Request) -> HttpResult { + assert!(req.method() == Method::Post, "Multipart request must use POST method!"); + + debug!("Fields: {}; Boundary: {}", self.fields[], self.boundary[]); + + if self.sized { + return self.send_sized(req); + } + + let Multipart { fields, boundary, ..} = self; + + apply_headers(&mut req, boundary[], None); + + debug!("{}", req.headers()); + + let mut req = try!(req.start()); + try!(write_body(&mut req, fields, boundary[])); + req.send() + } + + fn send_sized(self, mut req: Request) -> HttpResult { + let mut body: Vec = Vec::new(); + + let Multipart { fields, boundary, ..} = self; + + try!(write_body(&mut body, fields, boundary[])); + + apply_headers(&mut req, boundary[], Some(body.len())); + + let mut req = try!(req.start()); + try!(req.write(body[])); + req.send() + } +} + +fn apply_headers(req: &mut Request, boundary: &str, size: Option){ + let headers = req.headers_mut(); + + headers.set(ContentType(multipart_mime(boundary))); + + if let Some(size) = size { + headers.set(ContentLength(size)); + } +} + +fn write_body<'a>(wrt: &mut Writer, fields: Fields<'a>, boundary: &str) -> IoResult<()> { + try!(write_boundary(wrt, boundary[])); + + for (name, field) in fields.into_iter() { + try!(write_field(wrt, name, field, boundary)); + } + + Ok(()) +} + +fn write_field(wrt: &mut Writer, name: String, field: MultipartField, boundary: &str) -> IoResult<()> { + try!(write!(wrt, "Content-Disposition: form-data; name=\"{}\"\r\n\r\n", name)); + + try!(match field { + MultipartField::Text(text) => write_line(wrt, &*text), + MultipartField::File(file) => write_file(wrt, file), + }); + + write_boundary(wrt, boundary[]) +} + +fn write_boundary(wrt: &mut Writer, boundary: &str) -> IoResult<()> { + write!(wrt, "--{}\r\n", boundary) +} + +fn write_file(wrt: &mut Writer, mut file: MultipartFile) -> IoResult<()> { + try!(file.filename.map(|filename| write!(wrt, "; filename=\"{}\"\r\n", filename)).unwrap_or(Ok(()))); + try!(write!(wrt, "Content-Type: {}\r\n\r\n", file.content_type)); + ref_copy(&mut file.reader, wrt) +} + +/// Specialized write_line that writes CRLF after a line as per W3C specs +fn write_line(req: &mut Writer, s: &str) -> IoResult<()> { + req.write_str(s).and_then(|_| req.write(b"\r\n")) +} + +fn multipart_mime(bound: &str) -> Mime { + Mime( + TopLevel::Multipart, SubLevel::Ext("form-data".into_string()), + vec![(Attr::Ext("boundary".into_string()), Value::Ext(bound.into_string()))] + ) +} + + diff --git a/multipart/src/lib.rs b/multipart/src/lib.rs index e69de29bb..d028f95ac 100644 --- a/multipart/src/lib.rs +++ b/multipart/src/lib.rs @@ -0,0 +1,209 @@ +#![feature(if_let, slicing_syntax, default_type_params, phase, unboxed_closures, macro_rules)] +extern crate hyper; +#[phase(plugin, link)] extern crate log; + +extern crate mime; +extern crate serialize; + +use self::mime::Mime; + +use std::fmt::{Formatter, Show}; +use std::fmt::Error as FormatError; + +use std::io::{File, IoErrorKind, IoResult, TempDir}; + +use std::io::fs::PathExtensions; + +pub mod client; +pub mod server; +pub mod mime_guess; + +/// A representation of a file in HTTP `multipart/form-data`. +/// +/// This struct has an input "flavor" and an output "flavor". +/// The input "flavor" is used internally by `client::Multipart::add_file()` +/// and is never exposed to the user. +/// +/// The output "flavor" is returned by `server::Multipart::read_entry()` and represents +/// a file entry in the incoming multipart request. +/// +/// Note that in the output "flavor", the file is not yet saved to the system; +/// instead, the struct implements a `Reader` that points +/// to the beginning of the file's contents in the HTTP stream. +/// You can read it to EOF, or use one of the `save_*()` methods here +/// to save it to disk. +pub struct MultipartFile<'a> { + filename: Option, + content_type: Mime, + reader: &'a mut (Reader + 'a), + tmp_dir: Option<&'a str>, +} + +impl<'a> MultipartFile<'a> { + fn from_octet(filename: Option, reader: &'a mut Reader, cont_type: &str, tmp_dir: &'a str) -> MultipartFile<'a> { + MultipartFile { + filename: filename, + reader: reader, + content_type: from_str(cont_type).unwrap_or_else(mime_guess::octet_stream), + tmp_dir: Some(tmp_dir), + } + } + + fn from_file(filename: Option, reader: &'a mut File, mime: Mime) -> MultipartFile<'a> { + MultipartFile { + filename: filename, + reader: reader, + content_type: mime, + tmp_dir: None, + } + } + + /// Save this file to `path`, discarding the filename. + /// + /// If successful, the file can be found at `path`. + pub fn save_as(&mut self, path: &Path) -> IoResult<()> { + let mut file = try!(File::create(path)); + + ref_copy(self.reader, &mut file) + } + + /// Save this file in the directory described by `dir`, + /// appending `filename` if present, or a random string otherwise. + /// + /// Returns the created file's path on success. + /// + /// ###Panics + /// If `dir` does not represent a directory. + pub fn save_in(&mut self, dir: &Path) -> IoResult { + assert!(dir.is_dir(), "Given path is not a directory!"); + + let path = dir.join(self.dest_filename()); + + try!(self.save_as(&path)); + + Ok(path) + } + + /// Save this file in the temp directory `tmpdir` if supplied, + /// or a random subdirectory under `std::os::tmp_dir()` otherwise. + /// The same directory is used for all files in the same request). + /// + /// + /// Returns the created file's path on success. + pub fn save_temp(&mut self, tmp_dir: Option<&TempDir>) -> IoResult { + use std::os; + + let dir = match tmp_dir { + Some(tmp_dir) => tmp_dir.path().clone(), + None => os::tmpdir().join(self.tmp_dir.unwrap()), + }; + + self.save_in(&dir) + } + + fn dest_filename(&self) -> String { + self.filename.as_ref().map_or_else(|| random_alphanumeric(10), |s| s.clone()) + } + + pub fn filename(&self) -> Option<&str> { + self.filename.as_ref().map(|s| s[]) + } + + /// Get the content type of this file. + /// On the client, it is guessed by the file extension. + /// On the server, it is retrieved from the request or assumed to be + /// `application/octet-stream`. + pub fn content_type(&self) -> Mime { + self.content_type.clone() + } +} + +impl<'a> Show for MultipartFile<'a> { + fn fmt(&self, fmt: &mut Formatter) -> Result<(), FormatError> { + write!(fmt, "Filename: {} Content-Type: {}", self.filename, self.content_type) + } +} + + +/// A field in a `multipart/form-data` request. +/// +/// Like `MultipartFile`, this is used in both the client and server-side implementations, +/// but only exposed to the user on the server. +/// +/// This enum does not include the names of the fields, as those are yielded separately +/// by `server::Multipart::read_entry()`. +#[deriving(Show)] +pub enum MultipartField<'a> { + /// A text field. + Text(String), + /// A file field, including the content type and optional filename + /// along with a `Reader` implementation for getting the contents. + File(MultipartFile<'a>), + // MultiFiles(Vec), /* TODO: Multiple files */ +} + +impl<'a> MultipartField<'a> { + + /// Borrow this field as a text field, if possible. + pub fn as_text<'a>(&'a self) -> Option<&'a str> { + match *self { + MultipartField::Text(ref s) => Some(s[]), + _ => None, + } + } + + /// Take this field as a text field, if possible, + /// returning `self` otherwise. + pub fn to_text(self) -> Result> { + match self { + MultipartField::Text(s) => Ok(s), + _ => Err(self), + } + } + + /// Borrow this field as a file field, if possible + /// Mutably borrows so the contents can be read. + pub fn as_file<'b>(&'b mut self) -> Option<&'b mut MultipartFile<'a>> { + match *self { + MultipartField::File(ref mut file) => Some(file), + _ => None, + } + } + + /// Take this field as a file field if possible, + /// returning `self` otherwise. + pub fn to_file(self) -> Result, MultipartField<'a>> { + match self { + MultipartField::File(file) => Ok(file), + _ => Err(self), + } + } +} + +impl<'a> Reader for MultipartFile<'a> { + fn read(&mut self, buf: &mut [u8]) -> IoResult{ + self.reader.read(buf) + } +} + +/// A copy of `std::io::util::copy` that takes trait references +pub fn ref_copy(r: &mut Reader, w: &mut Writer) -> IoResult<()> { + let mut buf = [0, ..1024 * 64]; + + loop { + let len = match r.read(&mut buf) { + Ok(len) => len, + Err(ref e) if e.kind == IoErrorKind::EndOfFile => return Ok(()), + Err(e) => return Err(e), + }; + try!(w.write(buf[..len])); + } +} + +/// Generate a random alphanumeric sequence of length `len` +fn random_alphanumeric(len: uint) -> String { + use std::rand::{task_rng, Rng}; + + task_rng().gen_ascii_chars().map(|ch| ch.to_lowercase()).take(len).collect() +} + diff --git a/multipart/src/mime_guess.rs b/multipart/src/mime_guess.rs index e69de29bb..c6954abb2 100644 --- a/multipart/src/mime_guess.rs +++ b/multipart/src/mime_guess.rs @@ -0,0 +1,83 @@ +//! Guessing of MIME types by file extension. +//! +//! Extension-to-MIME-type mappings are loaded from a JSON file at runtime +//! and stored in a thread-local heap. To limit memory usage, avoid calling +//! functions in this module from several threads. +//! +//! TODO: Make mapping cache global in a safe manner. + +use mime::{Mime, TopLevel, SubLevel}; + +use serialize::json::{mod, Json}; + +use std::collections::HashMap; + +/// Guess the MIME type of the `Path` by its extension. +/// +/// **Guess** is the operative word here, as the contents of a file +/// may not necessarily match its MIME type/extension. +pub fn guess_mime_type(path: &Path) -> Mime { + let ext = path.extension_str().unwrap_or(""); + + get_mime_type(ext) +} + +/// Extract the extension of `filename` and guess its MIME type. +/// +/// If there is no extension, or the extension has no known MIME association, +/// `applicaton/octet-stream` is assumed. +pub fn guess_mime_type_filename(filename: &str) -> Mime { + let path = Path::new(filename); + + guess_mime_type(&path) +} + +const MIME_TYPES: &'static str = include_str!("../mime_types.json"); + +// Lazily initialized task-local hashmap +// TODO: Make this global since it's read-only after init +thread_local!(static MIMES: HashMap = load_mime_types()) + +/// Get the MIME type associated with a file extension. +/// +/// If there is no association for the extension, or `ext` is empty, +/// `application/octet-stream` is returned. +pub fn get_mime_type(ext: &str) -> Mime { + if ext.is_empty() { return octet_stream(); } + + MIMES.with(|cache| cache.get(ext).cloned()).unwrap_or_else(octet_stream) +} + +/// Load the known mime types from the MIME_TYPES json +fn load_mime_types() -> HashMap { + let map = if let Json::Object(map) = json::from_str(MIME_TYPES).unwrap() { map } + else { unreachable!("MIME types should be supplied as a map!"); }; + + map.into_iter().filter_map(to_mime_mapping).collect() +} + +fn to_mime_mapping(val: (String, json::Json)) -> Option<(String, Mime)> { + if let (st, Json::String(mime)) = val { + if st.char_at(0) == '_' { return None; } + + if let Some(mime) = from_str::(&*mime) { + return Some((st, mime)) + } + } + + None +} + +/// Get the MIME type for `application/octet-stream` (generic binary stream) +pub fn octet_stream() -> Mime { + Mime(TopLevel::Application, SubLevel::Ext("octet-stream".into_string()), Vec::new()) +} + +#[test] +fn test_mime_type_guessing() { + assert!(get_mime_type("gif").to_string() == "image/gif".to_string()); + assert!(get_mime_type("txt").to_string() == "text/plain".to_string()); + assert!(get_mime_type("blahblah").to_string() == "application/octet-stream".to_string()); +} + + diff --git a/multipart/src/server/handler.rs b/multipart/src/server/handler.rs index 30434de9b..27702701b 100644 --- a/multipart/src/server/handler.rs +++ b/multipart/src/server/handler.rs @@ -13,11 +13,11 @@ use super::Multipart; /// A convenient wrapper for `Multipart::from_request()`. pub struct Switch { normal: H, - multipart: M, + multipart: M, } impl Switch where H: Handler, M: MultipartHandler { - /// Create a new `Switch` instance where + /// Create a new `Switch` instance where /// `normal` handles normal Hyper requests and `multipart` handles Multipart requests pub fn new(normal: H, multipart: M) -> Switch { Switch { @@ -28,7 +28,7 @@ impl Switch where H: Handler, M: MultipartHandler { } impl Handler for Switch where H: Handler, M: MultipartHandler { - fn handle<'a>(&'a self, req: Request<'a>, res: Response<'a>) { + fn handle(&self, req: Request, res: Response) { match Multipart::from_request(req) { Ok(multi) => self.multipart.handle_multipart(multi, res), Err(req) => self.normal.handle(req, res), @@ -38,7 +38,7 @@ impl Handler for Switch where H: Handler, M: MultipartHandler { /// A trait defining a type that can handle an incoming multipart request. /// -/// Extends to unboxed closures of the type `Fn(Multipart, Response)`, +/// Extends to unboxed closures of the type `Fn(Multipart, Response)`, /// and subsequently static functions. /// /// Since `Multipart` implements `Deref`, you can still access @@ -50,13 +50,13 @@ pub trait MultipartHandler: Send + Sync { impl MultipartHandler for F where F: for<'a> Fn(Multipart<'a>, Response) + Send + Sync { fn handle_multipart<'a>(&self, multipart: Multipart<'a>, response: Response) { - (*self)(multipart, response); + (*self)(multipart, response); } } /// A container for an unboxed closure that implements `hyper::server::Handler`. /// -/// This exists because as of this writing, `Handler` is not automatically implemented for +/// This exists because as of this writing, `Handler` is not automatically implemented for /// compatible unboxed closures (though this will likely change). /// /// No private fields, instantiate directly. @@ -68,5 +68,8 @@ pub struct UnboxedHandler { impl Handler for UnboxedHandler where F: Fn(Request, Response) + Send + Sync { fn handle(&self, req: Request, res: Response) { (self.f)(req, res); - } + } } + + + diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index f0259054b..455c1c08b 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -1,34 +1,35 @@ //! The server-side implementation of `multipart/form-data` requests. //! -//! Use this when you are implementing a server on top of Hyper and want to +//! Use this when you are implementing a server on top of Hyper and want to //! to parse and serve POST `multipart/form-data` requests. -//! +//! //! See the `Multipart` struct for more info. -use hyper::header::ContentType; +use hyper::header::common::content_type::ContentType; use hyper::server::request::Request; use hyper::method::Method; use mime::{Mime, TopLevel, SubLevel, Attr, Value}; -use super::{IntoString, MultipartField, MultipartFile}; +use super::{MultipartField, MultipartFile}; use std::cmp; + use std::collections::HashMap; -use std::old_io::{IoError, IoResult, EndOfFile, standard_error, OtherIoError}; -use std::ops::Deref; + +use std::io::{IoError, IoResult, EndOfFile, standard_error, OtherIoError}; pub mod handler; fn is_multipart_formdata(req: &Request) -> bool { req.method == Method::Post && req.headers.get::().map_or(false, |ct| { let ContentType(ref mime) = *ct; - + debug!("Content-Type: {}", mime); match *mime { Mime(TopLevel::Multipart, SubLevel::FormData, _) => true, - _ => false, + _ => false, } }) } @@ -36,16 +37,16 @@ fn is_multipart_formdata(req: &Request) -> bool { fn get_boundary(ct: &ContentType) -> Option { let ContentType(ref mime) = *ct; let Mime(_, _, ref params) = *mime; - - params.iter().find(|&&(ref name, _)| - if let Attr::Ext(ref name) = *name { - "boundary" == &**name + + params.iter().find(|&&(ref name, _)| + if let Attr::Ext(ref name) = *name { + name[] == "boundary" } else { false } - ).and_then(|&(_, ref val)| - if let Value::Ext(ref val) = *val { - Some(val.clone()) + ).and_then(|&(_, ref val)| + if let Value::Ext(ref val) = *val { + Some(val.clone()) } else { None } - ) + ) } /// The server-side implementation of `multipart/form-data` requests. @@ -65,11 +66,11 @@ macro_rules! try_find( ($haystack:expr, $f:ident, $needle:expr, $err:expr, $line:expr) => ( try!($haystack.$f($needle).ok_or(line_error($err, $line.clone()))) ) -); +) impl<'a> Multipart<'a> { - /// If the given `Request` is a POST request of `Content-Type: multipart/form-data`, + /// If the given `Request` is a POST request of `Content-Type: multipart/form-data`, /// return the wrapped request as `Ok(Multipart)`, otherwise `Err(Request)`. /// /// See the `handler` submodule for a convenient wrapper for this function, @@ -92,8 +93,8 @@ impl<'a> Multipart<'a> { /// /// ##Warning /// If the last returned entry had contents of type `MultipartField::File`, - /// calling this again will discard any unread contents of that entry! - pub fn read_entry<'b>(&'b mut self) -> IoResult<(String, MultipartField<'b>)> { + /// calling this again will discard any unread contents of that entry! + pub fn read_entry(&'a mut self) -> IoResult<(String, MultipartField<'a>)> { try!(self.source.consume_boundary()); let (disp_type, field_name, filename) = try!(self.read_content_disposition()); @@ -104,12 +105,12 @@ impl<'a> Multipart<'a> { detail: Some(format!("Content-Disposition: {}", disp_type)), }); } - + if let Some(content_type) = try!(self.read_content_type()) { let _ = try!(self.source.read_line()); // Consume empty line - Ok((field_name, + Ok((field_name, MultipartField::File( - MultipartFile::from_octet(filename, &mut self.source, &content_type, &self.tmp_dir) + MultipartFile::from_octet(filename, &mut self.source, content_type[], self.tmp_dir[]) ) )) } else { @@ -118,50 +119,50 @@ impl<'a> Multipart<'a> { // The last two characters are "\r\n". // We can't do a simple trim because the content might be terminated // with line separators we want to preserve. - Ok((field_name, MultipartField::Text(text[..text.len() - 2].into_string()))) - } + Ok((field_name, MultipartField::Text(text[..text.len() - 2].into_string()))) + } } - + /// Call `f` for each entry in the multipart request. /// This is a substitute for `Multipart` implementing `Iterator`, /// since `Iterator::next()` can't use bound lifetimes. /// - /// See https://www.reddit.com/r/rust/comments/2lkk\4\isize/concrete_lifetime_vs_bound_lifetime/ - pub fn foreach_entry FnMut(String, MultipartField<'b>)>(&'a mut self, mut f: F) { + /// See https://www.reddit.com/r/rust/comments/2lkk4i/concrete_lifetime_vs_bound_lifetime/ + pub fn foreach_entry FnMut(String, MultipartField<'a>)>(&mut self, mut f: F) { loop { match self.read_entry() { Ok((name, field)) => f(name, field), - Err(err) => { + Err(err) => { if err.kind != EndOfFile { error!("Error reading Multipart: {}", err); } break; - }, - } - } - } - + }, + } + } + } + fn read_content_disposition(&mut self) -> IoResult<(String, String, Option)> { - let line = try!(self.source.read_line()); + let line = try!(self.source.read_line()); // Find the end of CONT_DISP in the line let disp_type = { const CONT_DISP: &'static str = "Content-Disposition:"; - let disp_idx = try_find!(&line, find_str, CONT_DISP, + let disp_idx = try_find!(line[], find_str, CONT_DISP, "Content-Disposition subheader not found!", line) + CONT_DISP.len(); - let disp_type_end = try_find!(line[disp_idx..], find, ';', + let disp_type_end = try_find!(line[disp_idx..], find, ';', "Error parsing Content-Disposition value!", line); line[disp_idx .. disp_idx + disp_type_end].trim().into_string() }; - + let field_name = { const NAME: &'static str = "name=\""; - let name_idx = try_find!(&line, find_str, NAME, + let name_idx = try_find!(line[], find_str, NAME, "Error parsing field name!", line) + NAME.len(); let name_end = try_find!(line[name_idx ..], find, '"', @@ -173,12 +174,12 @@ impl<'a> Multipart<'a> { let filename = { const FILENAME: &'static str = "filename=\""; - let filename_idx = line.find_str(FILENAME).map(|idx| idx + FILENAME.len()); + let filename_idx = line[].find_str(FILENAME).map(|idx| idx + FILENAME.len()); let filename_idxs = with(filename_idx, |&start| line[start ..].find('"')); - + filename_idxs.map(|(start, end)| line[start .. start + end].into_string()) }; - + Ok((disp_type, field_name, filename)) } @@ -187,10 +188,10 @@ impl<'a> Multipart<'a> { let line = try!(self.source.read_line()); const CONTENT_TYPE: &'static str = "Content-Type:"; - + let type_idx = (&*line).find_str(CONTENT_TYPE); - // FIXME Will not properly parse for multiple files! + // FIXME Will not properly parse for multiple files! // Does not expect boundary= Ok(type_idx.map(|start| line[start + CONTENT_TYPE.len()..].trim().into_string())) } @@ -200,7 +201,7 @@ impl<'a> Multipart<'a> { /// /// If `dir` is none, chooses a random subdirectory under `std::os::tmpdir()`. pub fn save_all(mut self, dir: Option<&Path>) -> IoResult { - let dir = dir.map_or_else(|| ::std::os::tmpdir().join(&self.tmp_dir), |path| path.clone()); + let dir = dir.map_or_else(|| ::std::os::tmpdir().join(self.tmp_dir[]), |path| path.clone()); let mut entries = Entries::with_path(dir); @@ -210,8 +211,8 @@ impl<'a> Multipart<'a> { Ok((name, MultipartField::File(mut file))) => { let path = try!(file.save_in(&entries.dir)); entries.files.insert(name, path); - }, - Err(err) => { + }, + Err(err) => { if err.kind != EndOfFile { error!("Error reading Multipart: {}", err); } @@ -225,24 +226,23 @@ impl<'a> Multipart<'a> { } } -impl<'a> Deref for Multipart<'a> { - type Target = Request<'a>; +impl<'a> Deref> for Multipart<'a> { fn deref(&self) -> &Request<'a> { - self.source.borrow_reader() - } + self.source.borrow_reader() + } } -fn with Option>(left: Option, right: F) -> Option<(T, U)> { +fn with(left: Option, right: |&T| -> Option) -> Option<(T, U)> { let temp = left.as_ref().and_then(right); match (left, temp) { (Some(lval), Some(rval)) => Some((lval, rval)), - _ => None, + _ => None, } -} +} fn line_error(msg: &'static str, line: String) -> IoError { - IoError { - kind: OtherIoError, + IoError { + kind: OtherIoError, desc: msg, detail: Some(line), } @@ -263,15 +263,15 @@ impl Entries { files: HashMap::new(), dir: path, } - } + } } /* FIXME: Can't have an iterator return a borrowed reference impl<'a> Iterator<(String, MultipartField<'a>)> for Multipart<'a> { fn next(&mut self) -> Option<(String, MultipartField<'a>)> { match self.read_entry() { - Ok(ok) => Some(ok), - Err(err) => { + Ok(ok) => Some(ok), + Err(err) => { if err.kind != EndOfFile { error!("Error reading Multipart: {}", err); } @@ -279,7 +279,7 @@ impl<'a> Iterator<(String, MultipartField<'a>)> for Multipart<'a> { None }, } - } + } } */ @@ -287,17 +287,17 @@ impl<'a> Iterator<(String, MultipartField<'a>)> for Multipart<'a> { pub struct BoundaryReader { reader: S, boundary: Vec, - last_search_idx: usize, + last_search_idx: uint, boundary_read: bool, buf: Vec, - buf_len: usize, + buf_len: uint, } fn eof() -> IoResult { - Err(standard_error(EndOfFile)) + Err(standard_error(EndOfFile)) } -const BUF_SIZE: usize = 1024 * 64; // 64k buffer +const BUF_SIZE: uint = 1024 * 64; // 64k buffer impl BoundaryReader where S: Reader { fn from_reader(reader: S, boundary: String) -> BoundaryReader { @@ -310,7 +310,7 @@ impl BoundaryReader where S: Reader { last_search_idx: 0, boundary_read: false, buf: buf, - buf_len: 0, + buf_len: 0, } } @@ -319,36 +319,36 @@ impl BoundaryReader where S: Reader { try!(self.true_fill_buf()); if self.buf_len == 0 { return eof(); } - - let lookahead = &self.buf[self.last_search_idx .. self.buf_len]; - + + let lookahead = self.buf[self.last_search_idx .. self.buf_len]; + let search_idx = lookahead.position_elem(&self.boundary[0]) .unwrap_or(lookahead.len() - 1); debug!("Search idx: {}", search_idx); self.boundary_read = lookahead[search_idx..] - .starts_with(&self.boundary); + .starts_with(self.boundary[]); self.last_search_idx += search_idx; if !self.boundary_read { - self.last_search_idx += 1; + self.last_search_idx += 1; } } else if self.last_search_idx == 0 { - return Err(standard_error(EndOfFile)) + return Err(standard_error(EndOfFile)) } - - Ok(()) + + Ok(()) } /// Read bytes until the reader is full fn true_fill_buf(&mut self) -> IoResult<()> { - let mut bytes_read = 1usize; - + let mut bytes_read = 1u; + while bytes_read != 0 { - bytes_read = match self.reader.read(&mut self.buf[self.buf_len..]) { + bytes_read = match self.reader.read(self.buf[mut self.buf_len..]) { Ok(read) => read, Err(err) => if err.kind == EndOfFile { break; } else { return Err(err); }, }; @@ -359,46 +359,46 @@ impl BoundaryReader where S: Reader { Ok(()) } - fn _consume(&mut self, amt: usize) { + fn _consume(&mut self, amt: uint) { use std::ptr::copy_memory; - + assert!(amt <= self.buf_len); let src = self.buf[amt..].as_ptr(); - let dest = self.buf.as_mut_ptr(); + let dest = self.buf[mut].as_mut_ptr(); unsafe { copy_memory(dest, src, self.buf_len - amt); } - + self.buf_len -= amt; - self.last_search_idx -= amt; + self.last_search_idx -= amt; } fn consume_boundary(&mut self) -> IoResult<()> { while !self.boundary_read { match self.read_to_boundary() { Ok(_) => (), - Err(e) => if e.kind == EndOfFile { - break; - } else { + Err(e) => if e.kind == EndOfFile { + break; + } else { return Err(e); } } } - + let consume_amt = cmp::min(self.buf_len, self.last_search_idx + self.boundary.len()); debug!("Consume amt: {} Buf len: {}", consume_amt, self.buf_len); self._consume(consume_amt); self.last_search_idx = 0; - self.boundary_read = false; - - Ok(()) + self.boundary_read = false; + + Ok(()) } #[allow(unused)] fn set_boundary(&mut self, boundary: String) { - self.boundary = boundary.into_bytes(); + self.boundary = boundary.into_bytes(); } pub fn borrow_reader<'a>(&'a self) -> &'a S { @@ -407,31 +407,31 @@ impl BoundaryReader where S: Reader { } impl Reader for BoundaryReader where S: Reader { - fn read(&mut self, buf: &mut [u8]) -> IoResult { - use std::cmp; + fn read(&mut self, buf: &mut [u8]) -> IoResult { + use std::cmp; use std::slice::bytes::copy_memory; - try!(self.read_to_boundary()); + try!(self.read_to_boundary()); let trunc_len = cmp::min(buf.len(), self.last_search_idx); - copy_memory(buf, &self.buf[..trunc_len]); + copy_memory(buf, self.buf[..trunc_len]); self._consume(trunc_len); - Ok(trunc_len) - } + Ok(trunc_len) + } } impl Buffer for BoundaryReader where S: Reader { fn fill_buf<'a>(&'a mut self) -> IoResult<&'a [u8]> { try!(self.read_to_boundary()); + + let buf = self.buf[..self.last_search_idx]; - let buf = &self.buf[..self.last_search_idx]; - - Ok(buf) + Ok(buf) } - fn consume(&mut self, amt: usize) { + fn consume(&mut self, amt: uint) { assert!(amt <= self.last_search_idx); self._consume(amt); } @@ -456,21 +456,22 @@ dashed-value-2\r debug!("Read 1"); let string = reader.read_to_string().unwrap(); debug!("{}", string); - assert!(string.trim().is_empty()); + assert!(string[].trim().is_empty()); debug!("Consume 1"); reader.consume_boundary().unwrap(); debug!("Read 2"); - assert_eq!(reader.read_to_string().unwrap().trim(), "dashed-value-1"); + assert_eq!(reader.read_to_string().unwrap()[].trim(), "dashed-value-1"); debug!("Consume 2"); reader.consume_boundary().unwrap(); debug!("Read 3"); - assert_eq!(reader.read_to_string().unwrap().trim(), "dashed-value-2"); + assert_eq!(reader.read_to_string().unwrap()[].trim(), "dashed-value-2"); debug!("Consume 3"); reader.consume_boundary().unwrap(); - + } + diff --git a/multipart/tests/multipart_server.rs b/multipart/tests/multipart_server.rs index b808ced2c..67daafba4 100644 --- a/multipart/tests/multipart_server.rs +++ b/multipart/tests/multipart_server.rs @@ -1,31 +1,34 @@ +#![feature(unboxed_closures, if_let, slicing_syntax)] #![allow(dead_code)] extern crate hyper; extern crate multipart; -use hyper::server::{Listening, Server, Request, Response}; -use hyper::client::Request as ClientReq; -use hyper::status::StatusCode; -use hyper::Url; +use self::hyper::server::{Listening, Server, Request, Response}; +use self::hyper::client::Request as ClientReq; +use self::hyper::status::StatusCode; +use self::hyper::Url; -use multipart::server::Multipart; -use multipart::client::Multipart as ClientMulti; -use std::old_io::net::ip::Ipv4Addr; +use self::multipart::server::Multipart; + +use self::multipart::client::Multipart as ClientMulti; + +use std::io::net::ip::Ipv4Addr; use std::rand::random; -fn ok_serv(req: Request, mut res: Response) { +fn ok_serv(req: Request, mut res: Response) { let mut multipart = Multipart::from_request(req).ok().expect("Could not create multipart!"); multipart.foreach_entry(|&: _, _| ()); - + *res.status_mut() = StatusCode::Ok; res.start().unwrap().end().unwrap(); } -thread_local!(static PORT: u16 = random()); +thread_local!(static PORT: u16 = random()) fn server() -> Listening { let server = PORT.with(|port| Server::http(Ipv4Addr(127, 0, 0, 1), *port)); @@ -36,7 +39,7 @@ fn server() -> Listening { fn client_api_test() { let mut server = server(); - let address = PORT.with(|port| format!("http://localhost:{}/", port)); + let address = PORT.with(|port| format!("http://localhost:{}/", port)); let request = ClientReq::post(Url::parse(&*address).unwrap()).unwrap(); @@ -47,6 +50,7 @@ fn client_api_test() { multipart.sized = true; multipart.send(request).unwrap(); - + server.close().unwrap(); } + From 41774e474c9ca63cd601f8dff3ebd42bfb1127e8 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sat, 4 Jul 2015 16:23:12 -0700 Subject: [PATCH 048/453] WIP revival (does not compile) --- multipart/Cargo.toml | 5 +- multipart/src/client.rs | 240 +++++++++++++++++++++ multipart/src/lib.rs | 208 ++++++++++++++++++ multipart/src/{server/mod.rs => server.rs} | 90 ++++---- 4 files changed, 499 insertions(+), 44 deletions(-) rename multipart/src/{server/mod.rs => server.rs} (85%) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 25e68f2ad..f08390ec0 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -17,8 +17,11 @@ keywords = ["form-data", "hyper", "http", "post", "upload"] [dependencies] log = "*" mime = "*" +mime_guess = "*" rand = "*" rustc-serialize = "*" [dependencies.hyper] -git = "https://github.com/hyperium/hyper" +version = "*" +optional = true + diff --git a/multipart/src/client.rs b/multipart/src/client.rs index e69de29bb..362a1d975 100644 --- a/multipart/src/client.rs +++ b/multipart/src/client.rs @@ -0,0 +1,240 @@ +//! The client side implementation of `multipart/form-data` requests. +//! +//! Use this when sending POST requests with files to a server. +//! +//! See the `Multipart` struct for more info. +use mime::{Mime, TopLevel, SubLevel, Attr, Value}; + +use mime_guess::{guess_mime_type, octet_stream}; + +use std::borrow::Cow; + +use std::fs::File; +use std::io; +use std::io::prelude::*; + +use super::{MultipartField, MultipartFile, ref_copy, random_alphanumeric}; + +const BOUNDARY_LEN: usize = 8; + +type Fields<'a> = Vec<(Cow<'a, str>, MultipartField<'a>)>; + +/// The entry point of the client-side multipart API. +/// +/// Add text fields with `.add_text()` and files with `.add_file()`, +/// then obtain a `hyper::client::Request` object and pass it to `.send()`. +pub struct Multipart<'a> { + fields: Fields<'a>, + boundary: String, + /// If the request can be sized. + /// If true, avoid using chunked requests. + /// Defaults to `false`. + pub sized: bool, +} + +impl<'a> Multipart<'a> { + + /// Create a new `Multipart` instance with an empty set of fields. + pub fn new() -> Multipart<'a> { + Multipart { + fields: Vec::new(), + boundary: random_alphanumeric(BOUNDARY_LEN), + sized: false, + } + } + + /// Add a text field to this multipart request. + /// `name` and `val` can be either owned `String` or `&str`. + /// Prefer `String` if you're trying to limit allocations and copies. + pub fn add_text>, V: Into>>(&mut self, name: N, val: V) { + self.add_field(name, MultipartField::Text(val)); + } + + /// Add the file to the multipart request, guessing its `Content-Type` + /// from its extension and supplying its filename. + /// + /// See `add_stream()`. + pub fn add_file>>(&mut self, name: N, file: &'a mut File) { + let filename = file.path().filename_str().map(|s| s.into_string()); + let content_type = guess_mime_type(file.path()); + + self.add_field(name, + MultipartField::File(MultipartFile::from_file(filename, file, content_type)) + ); + } + + /// Add a `Read` as a file field, supplying `filename` if given, + /// and `content_type` if given or `application/octet-stream` if not. + /// + /// ##Warning + /// The given `Read` **must** be able to read to EOF (end of file/no more data). + /// If it never returns EOF it will be read to infinity (even if it reads 0 bytes forever) + /// and the request will never be completed. + /// + /// If `sized` is `true`, this adds an additional consequence of out-of-control + /// memory usage, as `Multipart` tries to read an infinite amount of data into memory. + /// + /// Use `std::io::util::LimitReader` if you wish to send data from a `Read` + /// that will never return EOF otherwise. + pub fn add_stream>>(&mut self, name: N, reader: &'a mut (Read + 'a), + filename: Option, content_type: Option) { + self.add_field(name, + MultipartField::File(MultipartFile { + filename: filename, + content_type: content_type.unwrap_or_else(octet_stream), + reader: reader, + tmp_dir: None, + }) + ); + } + + fn add_field>>(&mut self, name: N, val: MultipartField<'a>) { + self.fields.push((name.into_string(), val)); + } + + /// Apply the appropriate headers to the `Request` (obtained from Hyper) and send the data. + /// If `self.sized == true`, send a sized (non-chunked) request, setting the `Content-Length` + /// header. Else, send a chunked request. + /// + /// Sized requests are more human-readable and use less bandwidth + /// (as chunking adds [significant visual noise and overhead][chunked-example]), + /// but they must be able to load their entirety, including the contents of all files + /// and streams, into memory so the request body can be measured and its size set + /// in the `Content-Length` header. + /// + /// Prefer chunked requests when sending very large or numerous files, + /// or when human-readability or bandwidth aren't an issue. + /// + /// [chunked-example]: http://en.wikipedia.org/wiki/Chunked_transfer_encoding#Example + /// + /// ##Panics + /// If `req` fails sanity checks in `HttpRequest::apply_headers()`. + pub fn send(self, mut req: R) -> R::RequestResult where R: HttpRequest { + debug!("Fields: {}; Boundary: {}", self.fields, self.boundary); + + if self.sized { + return self.send_sized(req); + } + + let Multipart { fields, boundary, ..} = self; + + req.apply_headers(&boundary, None); + + req.send(|req| write_body(&mut req, fields, &boundary)) + } + + fn send_sized(self, mut req: R) -> R::RequestResult where R: HttpRequest { + let mut body: Vec = Vec::new(); + + let Multipart { fields, boundary, ..} = self; + + try!(write_body(&mut body, fields, boundary)); + + req.apply_headers(&boundary, Some(body.len())); + req.send(|req| req.write(&body)) + } +} + + +fn write_body(wrt: &mut W, fields: Fields, boundary: &str) -> io::Result<()> { + try!(write_boundary(wrt, boundary)); + + for (name, field) in fields.into_iter() { + try!(write_field(wrt, name, field, boundary)); + } + + Ok(()) +} + +fn write_field(wrt: &mut Write, name: String, field: MultipartField, boundary: &str) -> io::Result<()> { + try!(write!(wrt, "Content-Disposition: form-data; name=\"{}\"\r\n\r\n", name)); + + try!(match field { + MultipartField::Text(text) => write_line(wrt, &*text), + MultipartField::File(file) => write_file(wrt, file), + }); + + write_boundary(wrt, boundary) +} + +fn write_boundary(wrt: &mut Write, boundary: &str) -> io::Result<()> { + write!(wrt, "--{}\r\n", boundary) +} + +fn write_file(wrt: &mut Write, mut file: MultipartFile) -> io::Result<()> { + try!(file.filename.map(|filename| write!(wrt, "; filename=\"{}\"\r\n", filename)).unwrap_or(Ok(()))); + try!(write!(wrt, "Content-Type: {}\r\n\r\n", file.content_type)); + ref_copy(&mut file.reader, wrt) +} + +/// Specialized write_line that writes CRLF after a line as per W3C specs +fn write_line(req: &mut Write, s: &str) -> io::Result<()> { + req.write_str(s).and_then(|_| req.write(b"\r\n")) +} + +fn multipart_mime(bound: &str) -> Mime { + Mime( + TopLevel::Multipart, SubLevel::Ext("form-data".into_string()), + vec![(Attr::Ext("boundary".into_string()), Value::Ext(bound.into_string()))] + ) +} + +pub trait HttpRequest { + type RequestStream: Write; + type Response; + type RequestErr: From; + type RequestResult = Result; + + /// Set the `ContentType` header to `multipart/form-data` and supply the `boundary` value. + /// + /// If `content_len` is given, set the `ContentLength` header to its value. + fn apply_headers(&mut self, boundary: &str, content_len: Option); + /// Open the request stream and invoke the given closure, where the request body will be + /// written. After the closure returns, finalize the request and return its result. + fn send(self, send_fn: F) -> Self::RequestResult + where F: FnOnce(&mut Self::RequestStream) -> io::Result<()>; +} + +#[cfg(feature = "hyper")] +mod hyper_impl { + use hyper::client::request::Request; + use hyper::client::response::Response; + use hyper::error::Error as HyperError; + use hyper::net::{Fresh, Streaming}; + + use std::io; + + impl super::HttpRequest for Request { + type RequestStream = Request; + type Response = Response; + type RequestErr = HyperError; + + /// #Panics + /// If the `Request` method is not `Method::Post`. + fn apply_headers(&mut self, boundary: &str, content_len: Option) { + use hyper::header::{ContentType, ContentLength}; + use hyper::method::Method; + + assert!(self.method() == Method::Post, "Multipart request must use POST method!"); + + let headers = self.headers_mut(); + + headers.set(ContentType(super::multipart_mime(boundary))); + + if let Some(size) = content_len { + headers.set(ContentLength(size)); + } + + debug!("Hyper headers: {}", self.headers()); + } + + fn send(self, send_fn: F) -> Self::RequestResult + where F: FnOnce(&mut Request) -> io::Result<()> + { + let mut req = try!(self.start()); + try!(send_fn(&mut req)); + req.send() + } + } +} + diff --git a/multipart/src/lib.rs b/multipart/src/lib.rs index e69de29bb..487bed1b6 100644 --- a/multipart/src/lib.rs +++ b/multipart/src/lib.rs @@ -0,0 +1,208 @@ +#[macro_use] extern crate log; + +extern crate mime; +extern crate mime_guess; +extern crate rand; +extern crate rustc_serialize; + +#[cfg(feature = "hyper")] +extern crate hyper; + +use mime::Mime; + +use std::borrow::Cow; +use std::fmt; + +use std::fs::File; +use std::io; +use std::io::prelude::*; +use std::path::{Path, PathBuf}; + + +pub mod client; +pub mod server; + +/// A representation of a file in HTTP `multipart/form-data`. +/// +/// This struct has an input "flavor" and an output "flavor". +/// The input "flavor" is used internally by `client::Multipart::add_file()` +/// and is never exposed to the user. +/// +/// The output "flavor" is returned by `server::Multipart::read_entry()` and represents +/// a file entry in the incoming multipart request. +/// +/// Note that in the output "flavor", the file is not yet saved to the system; +/// instead, the struct implements a `Reader` that points +/// to the beginning of the file's contents in the HTTP stream. +/// You can read it to EOF, or use one of the `save_*()` methods here +/// to save it to disk. +pub struct MultipartFile<'a> { + filename: Option, + content_type: Mime, + reader: &'a mut (Read + 'a), + tmp_dir: Option<&'a str>, +} + +impl<'a> MultipartFile<'a> { + fn from_octet(filename: Option, reader: &'a mut Read, cont_type: &str, tmp_dir: &'a str) -> MultipartFile<'a> { + MultipartFile { + filename: filename, + reader: reader, + content_type: cont_type.parse().unwrap_or_else(mime_guess::octet_stream), + tmp_dir: Some(tmp_dir), + } + } + + fn from_file(filename: Option, reader: &'a mut File, mime: Mime) -> MultipartFile<'a> { + MultipartFile { + filename: filename, + reader: reader, + content_type: mime, + tmp_dir: None, + } + } + + /// Save this file to `path`, discarding the filename. + /// + /// If successful, the file can be found at `path`. + pub fn save_as(&mut self, path: &Path) -> io::Result<()> { + let mut file = try!(File::create(path)); + + ref_copy(self.reader, &mut file) + } + + /// Save this file in the directory described by `dir`, + /// appending `filename` if present, or a random string otherwise. + /// + /// Returns the created file's path on success. + /// + /// ###Panics + /// If `dir` does not represent a directory. + pub fn save_in(&mut self, dir: &Path) -> io::Result { + assert!(dir.is_dir(), "Given path is not a directory!"); + + let path = dir.join(self.dest_filename()); + + try!(self.save_as(&path)); + + Ok(path) + } + + /// Save this file in the OS temp directory, returned from `std::env::temp_dir()`. + /// + /// Returns the created file's path on success. + pub fn save_temp(&mut self) -> io::Result { + use std::env; + + self.save_in(&env::temp_dir()) + } + + fn dest_filename(&self) -> String { + self.filename.as_ref().map_or_else(|| random_alphanumeric(10), |s| s.clone()) + } + + pub fn filename(&self) -> Option<&str> { + self.filename.as_ref().map(|s| s) + } + + /// Get the content type of this file. + /// On the client, it is guessed by the file extension. + /// On the server, it is retrieved from the request or assumed to be + /// `application/octet-stream`. + pub fn content_type(&self) -> Mime { + self.content_type.clone() + } +} + +impl<'a> fmt::Debug for MultipartFile<'a> { + fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result<()> { + write!(fmt, "Filename: {} Content-Type: {}", self.filename, self.content_type) + } +} + + +/// A field in a `multipart/form-data` request. +/// +/// Like `MultipartFile`, this is used in both the client and server-side implementations, +/// but only exposed to the user on the server. +/// +/// This enum does not include the names of the fields, as those are yielded separately +/// by `server::Multipart::read_entry()`. +#[derive(Debug)] +pub enum MultipartField<'a> { + /// A text field. + Text(Cow<'a, str>), + /// A file field, including the content type and optional filename + /// along with a `Read` implementation for getting the contents. + File(MultipartFile<'a>), + // MultiFiles(Vec), /* TODO: Multiple files */ +} + +impl<'a> MultipartField<'a> { + + /// Borrow this field as a text field, if possible. + pub fn as_text(&self) -> Option<&str> { + match *self { + MultipartField::Text(ref s) => Some(s), + _ => None, + } + } + + /// Take this field as a text field, if possible, + /// returning `self` otherwise. + pub fn to_text(self) -> Result, MultipartField> { + match self { + MultipartField::Text(s) => Ok(s), + _ => Err(self), + } + } + + /// Borrow this field as a file field, if possible + /// Mutably borrows so the contents can be read. + pub fn as_file<'b>(&'b mut self) -> Option<&'b mut MultipartFile<'a>> { + match *self { + MultipartField::File(ref mut file) => Some(file), + _ => None, + } + } + + /// Take this field as a file field if possible, + /// returning `self` otherwise. + pub fn to_file(self) -> Result, MultipartField<'a>> { + match self { + MultipartField::File(file) => Ok(file), + _ => Err(self), + } + } +} + +impl<'a> Read for MultipartFile<'a> { + fn read(&mut self, buf: &mut [u8]) -> io::Result{ + self.reader.read(buf) + } +} + +/// A copy of `std::io::util::copy` that takes trait references +pub fn ref_copy(r: &mut Read, w: &mut Write) -> io::Result<()> { + let mut buf = [0, ..1024 * 64]; + + loop { + let len = match r.read(&mut buf) { + Ok(len) => len, + Err(ref e) if e.kind() == io::ErrorKind::EndOfFile => return Ok(()), + Err(e) => return Err(e), + }; + try!(w.write(&buf[..len])); + } +} + +/// Generate a random alphanumeric sequence of length `len` +fn random_alphanumeric(len: usize) -> String { + use rand::Rng; + + rand::thread_rng().gen_ascii_chars().map(|ch| ch.to_lowercase()).take(len).collect() +} + +fn is_error_eof(err: &io::Error) -> bool { + err.kind() == io::ErrorKind::EndOfFile +} diff --git a/multipart/src/server/mod.rs b/multipart/src/server.rs similarity index 85% rename from multipart/src/server/mod.rs rename to multipart/src/server.rs index f0259054b..7e8cd4982 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server.rs @@ -11,14 +11,16 @@ use hyper::method::Method; use mime::{Mime, TopLevel, SubLevel, Attr, Value}; -use super::{IntoString, MultipartField, MultipartFile}; +use super::{MultipartField, MultipartFile}; use std::cmp; use std::collections::HashMap; -use std::old_io::{IoError, IoResult, EndOfFile, standard_error, OtherIoError}; use std::ops::Deref; -pub mod handler; +use std::io; +use std::io::prelude::*; + +use std::path::{Path, PathBuf}; fn is_multipart_formdata(req: &Request) -> bool { req.method == Method::Post && req.headers.get::().map_or(false, |ct| { @@ -93,16 +95,15 @@ impl<'a> Multipart<'a> { /// ##Warning /// If the last returned entry had contents of type `MultipartField::File`, /// calling this again will discard any unread contents of that entry! - pub fn read_entry<'b>(&'b mut self) -> IoResult<(String, MultipartField<'b>)> { + pub fn read_entry<'b>(&'b mut self) -> io::Result<(String, MultipartField<'b>)> { try!(self.source.consume_boundary()); let (disp_type, field_name, filename) = try!(self.read_content_disposition()); - if &*disp_type != "form-data" { - return Err(IoError { - kind: OtherIoError, - desc: "Content-Disposition value was not \"form-data\"", - detail: Some(format!("Content-Disposition: {}", disp_type)), - }); + if disp_type != "form-data" { + return Err(io::Error::new( + io::ErrorKind::OtherIoError, + format!("Content-Disposition value: {:?} expected: \"form-data\"", disp_type), + )); } if let Some(content_type) = try!(self.read_content_type()) { @@ -132,7 +133,7 @@ impl<'a> Multipart<'a> { match self.read_entry() { Ok((name, field)) => f(name, field), Err(err) => { - if err.kind != EndOfFile { + if err.kind() != io::ErrorKind::EndOfFile { error!("Error reading Multipart: {}", err); } @@ -142,7 +143,7 @@ impl<'a> Multipart<'a> { } } - fn read_content_disposition(&mut self) -> IoResult<(String, String, Option)> { + fn read_content_disposition(&mut self) -> io::Result<(String, String, Option)> { let line = try!(self.source.read_line()); // Find the end of CONT_DISP in the line @@ -182,7 +183,7 @@ impl<'a> Multipart<'a> { Ok((disp_type, field_name, filename)) } - fn read_content_type(&mut self) -> IoResult> { + fn read_content_type(&mut self) -> io::Result> { debug!("Read content type!"); let line = try!(self.source.read_line()); @@ -195,12 +196,12 @@ impl<'a> Multipart<'a> { Ok(type_idx.map(|start| line[start + CONTENT_TYPE.len()..].trim().into_string())) } - /// Read the request fully, parsing all fields and saving all files to the given directory or a - /// temporary, and return the result. + /// Read the request fully, parsing all fields and saving all files + /// to the given directory (if given) and return the result. /// /// If `dir` is none, chooses a random subdirectory under `std::os::tmpdir()`. - pub fn save_all(mut self, dir: Option<&Path>) -> IoResult { - let dir = dir.map_or_else(|| ::std::os::tmpdir().join(&self.tmp_dir), |path| path.clone()); + pub fn save_all(mut self, dir: Option<&Path>) -> io::Result { + let dir = dir.map_or_else(|| ::std::env::temp_dir().join(&self.tmp_dir), |path| path.clone()); let mut entries = Entries::with_path(dir); @@ -212,7 +213,7 @@ impl<'a> Multipart<'a> { entries.files.insert(name, path); }, Err(err) => { - if err.kind != EndOfFile { + if super::is_error_eof(&err) { error!("Error reading Multipart: {}", err); } @@ -240,12 +241,11 @@ fn with Option>(left: Option, right: F) -> Option<( } } -fn line_error(msg: &'static str, line: String) -> IoError { - IoError { - kind: OtherIoError, - desc: msg, - detail: Some(line), - } +fn line_error(msg: &str, line: String) -> io::Error { + io::Error::new( + io::ErrorKind::Other, + format!("Error: {:?} on line of request: {:?}", msg, line) + ) } /// A result of `Multipart::save_all()`. @@ -283,7 +283,7 @@ impl<'a> Iterator<(String, MultipartField<'a>)> for Multipart<'a> { } */ -/// A `Reader` that will yield bytes until it sees a given sequence. +/// A struct implementing `Read` that will yield bytes until it sees a given sequence. pub struct BoundaryReader { reader: S, boundary: Vec, @@ -293,13 +293,13 @@ pub struct BoundaryReader { buf_len: usize, } -fn eof() -> IoResult { - Err(standard_error(EndOfFile)) +fn eof() -> io::Result { + Err(io::Error::new(io::ErrorKind::EndOfFile, "End of the stream was reached!")) } const BUF_SIZE: usize = 1024 * 64; // 64k buffer -impl BoundaryReader where S: Reader { +impl BoundaryReader where S: Read { fn from_reader(reader: S, boundary: String) -> BoundaryReader { let mut buf = Vec::with_capacity(BUF_SIZE); unsafe { buf.set_len(BUF_SIZE); } @@ -314,7 +314,7 @@ impl BoundaryReader where S: Reader { } } - fn read_to_boundary(&mut self) -> IoResult<()> { + fn read_to_boundary(&mut self) -> io::Result<()> { if !self.boundary_read { try!(self.true_fill_buf()); @@ -336,21 +336,20 @@ impl BoundaryReader where S: Reader { self.last_search_idx += 1; } + Ok(()) } else if self.last_search_idx == 0 { - return Err(standard_error(EndOfFile)) + eof() } - - Ok(()) } /// Read bytes until the reader is full - fn true_fill_buf(&mut self) -> IoResult<()> { + fn true_fill_buf(&mut self) -> io::Result<()> { let mut bytes_read = 1usize; while bytes_read != 0 { bytes_read = match self.reader.read(&mut self.buf[self.buf_len..]) { Ok(read) => read, - Err(err) => if err.kind == EndOfFile { break; } else { return Err(err); }, + Err(err) => if super::is_error_eof(&err) { break; } else { return Err(err); }, }; self.buf_len += bytes_read; @@ -360,27 +359,27 @@ impl BoundaryReader where S: Reader { } fn _consume(&mut self, amt: usize) { - use std::ptr::copy_memory; + use std::ptr; assert!(amt <= self.buf_len); let src = self.buf[amt..].as_ptr(); let dest = self.buf.as_mut_ptr(); - unsafe { copy_memory(dest, src, self.buf_len - amt); } + unsafe { ptr::copy(dest, src, self.buf_len - amt); } self.buf_len -= amt; self.last_search_idx -= amt; } - fn consume_boundary(&mut self) -> IoResult<()> { + fn consume_boundary(&mut self) -> io::Result<()> { while !self.boundary_read { match self.read_to_boundary() { Ok(_) => (), - Err(e) => if e.kind == EndOfFile { + Err(err) => if super::is_error_eof(&err) { break; } else { - return Err(e); + return Err(err); } } } @@ -406,8 +405,8 @@ impl BoundaryReader where S: Reader { } } -impl Reader for BoundaryReader where S: Reader { - fn read(&mut self, buf: &mut [u8]) -> IoResult { +impl Read for BoundaryReader where S: Read { + fn read(&mut self, buf: &mut [u8]) -> io::Result { use std::cmp; use std::slice::bytes::copy_memory; @@ -422,8 +421,8 @@ impl Reader for BoundaryReader where S: Reader { } } -impl Buffer for BoundaryReader where S: Reader { - fn fill_buf<'a>(&'a mut self) -> IoResult<&'a [u8]> { +impl BufRead for BoundaryReader where S: Read { + fn fill_buf(&mut self) -> io::Result<&[u8]> { try!(self.read_to_boundary()); let buf = &self.buf[..self.last_search_idx]; @@ -437,6 +436,11 @@ impl Buffer for BoundaryReader where S: Reader { } } +pub trait HttpRequest: Read { + +} + + #[test] fn test_boundary() { use std::io::BufReader; From 7853852255e05aeef1e3265915373af8868ba00e Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sat, 4 Jul 2015 16:25:05 -0700 Subject: [PATCH 049/453] Use external `mime_guess` --- multipart/src/mime_guess.rs | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 multipart/src/mime_guess.rs diff --git a/multipart/src/mime_guess.rs b/multipart/src/mime_guess.rs deleted file mode 100644 index e69de29bb..000000000 From 62cd4a3f56518eec444531118d56ba2326c95821 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sun, 5 Jul 2015 03:10:30 -0700 Subject: [PATCH 050/453] More WIP, bedtime --- multipart/.gitignore | 3 + multipart/src/client.rs | 11 +- multipart/src/lib.rs | 36 +- multipart/src/server.rs | 480 ------------------ multipart/src/server/boundary.rs | 195 +++++++ multipart/src/server/{handler.rs => hyper.rs} | 31 ++ multipart/src/server/mod.rs | 274 ++++++++++ 7 files changed, 524 insertions(+), 506 deletions(-) delete mode 100644 multipart/src/server.rs create mode 100644 multipart/src/server/boundary.rs rename multipart/src/server/{handler.rs => hyper.rs} (75%) create mode 100644 multipart/src/server/mod.rs diff --git a/multipart/.gitignore b/multipart/.gitignore index 8eea62b9d..f79cd6203 100644 --- a/multipart/.gitignore +++ b/multipart/.gitignore @@ -1,6 +1,7 @@ /target /Cargo.lock *.swp +*~ # Compiled files *.o *.so @@ -12,3 +13,5 @@ # Generated by Cargo /target/ + + diff --git a/multipart/src/client.rs b/multipart/src/client.rs index 362a1d975..b8ab6aee0 100644 --- a/multipart/src/client.rs +++ b/multipart/src/client.rs @@ -47,7 +47,7 @@ impl<'a> Multipart<'a> { /// `name` and `val` can be either owned `String` or `&str`. /// Prefer `String` if you're trying to limit allocations and copies. pub fn add_text>, V: Into>>(&mut self, name: N, val: V) { - self.add_field(name, MultipartField::Text(val)); + self.add_field(name, MultipartField::Text(val.into())); } /// Add the file to the multipart request, guessing its `Content-Type` @@ -55,10 +55,11 @@ impl<'a> Multipart<'a> { /// /// See `add_stream()`. pub fn add_file>>(&mut self, name: N, file: &'a mut File) { - let filename = file.path().filename_str().map(|s| s.into_string()); + let filename = file.path().filename_str().map(|s| s.to_owned()); let content_type = guess_mime_type(file.path()); - self.add_field(name, + self.add_field( + name, MultipartField::File(MultipartFile::from_file(filename, file, content_type)) ); } @@ -110,7 +111,7 @@ impl<'a> Multipart<'a> { /// ##Panics /// If `req` fails sanity checks in `HttpRequest::apply_headers()`. pub fn send(self, mut req: R) -> R::RequestResult where R: HttpRequest { - debug!("Fields: {}; Boundary: {}", self.fields, self.boundary); + debug!("Fields: {:?}; Boundary: {:?}", self.fields, self.boundary); if self.sized { return self.send_sized(req); @@ -128,7 +129,7 @@ impl<'a> Multipart<'a> { let Multipart { fields, boundary, ..} = self; - try!(write_body(&mut body, fields, boundary)); + try!(write_body(&mut body, fields, &boundary)); req.apply_headers(&boundary, Some(body.len())); req.send(|req| req.write(&body)) diff --git a/multipart/src/lib.rs b/multipart/src/lib.rs index 487bed1b6..7b3f64cf0 100644 --- a/multipart/src/lib.rs +++ b/multipart/src/lib.rs @@ -18,7 +18,6 @@ use std::io; use std::io::prelude::*; use std::path::{Path, PathBuf}; - pub mod client; pub mod server; @@ -40,16 +39,14 @@ pub struct MultipartFile<'a> { filename: Option, content_type: Mime, reader: &'a mut (Read + 'a), - tmp_dir: Option<&'a str>, } impl<'a> MultipartFile<'a> { - fn from_octet(filename: Option, reader: &'a mut Read, cont_type: &str, tmp_dir: &'a str) -> MultipartFile<'a> { + fn from_octet(filename: Option, reader: &'a mut Read, cont_type: &str) -> MultipartFile<'a> { MultipartFile { filename: filename, reader: reader, - content_type: cont_type.parse().unwrap_or_else(mime_guess::octet_stream), - tmp_dir: Some(tmp_dir), + content_type: cont_type.parse::().ok().unwrap_or_else(mime_guess::octet_stream), } } @@ -58,7 +55,6 @@ impl<'a> MultipartFile<'a> { filename: filename, reader: reader, content_type: mime, - tmp_dir: None, } } @@ -102,7 +98,7 @@ impl<'a> MultipartFile<'a> { } pub fn filename(&self) -> Option<&str> { - self.filename.as_ref().map(|s| s) + self.filename.as_ref().map(String::as_ref) } /// Get the content type of this file. @@ -115,8 +111,8 @@ impl<'a> MultipartFile<'a> { } impl<'a> fmt::Debug for MultipartFile<'a> { - fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result<()> { - write!(fmt, "Filename: {} Content-Type: {}", self.filename, self.content_type) + fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + write!(fmt, "Filename: {:?} Content-Type: {:?}", self.filename, self.content_type) } } @@ -150,7 +146,7 @@ impl<'a> MultipartField<'a> { /// Take this field as a text field, if possible, /// returning `self` otherwise. - pub fn to_text(self) -> Result, MultipartField> { + pub fn to_text(self) -> Result, MultipartField<'a>> { match self { MultipartField::Text(s) => Ok(s), _ => Err(self), @@ -159,7 +155,7 @@ impl<'a> MultipartField<'a> { /// Borrow this field as a file field, if possible /// Mutably borrows so the contents can be read. - pub fn as_file<'b>(&'b mut self) -> Option<&'b mut MultipartFile<'a>> { + pub fn as_file(&mut self) -> Option<&mut MultipartFile<'a>> { match *self { MultipartField::File(ref mut file) => Some(file), _ => None, @@ -184,25 +180,23 @@ impl<'a> Read for MultipartFile<'a> { /// A copy of `std::io::util::copy` that takes trait references pub fn ref_copy(r: &mut Read, w: &mut Write) -> io::Result<()> { - let mut buf = [0, ..1024 * 64]; + let mut buf = [0; 1024 * 64]; loop { - let len = match r.read(&mut buf) { - Ok(len) => len, - Err(ref e) if e.kind() == io::ErrorKind::EndOfFile => return Ok(()), - Err(e) => return Err(e), - }; + let len = try!(r.read(&mut buf)); + + if len == 0 { break; } + try!(w.write(&buf[..len])); } + + Ok(()) } /// Generate a random alphanumeric sequence of length `len` fn random_alphanumeric(len: usize) -> String { use rand::Rng; - rand::thread_rng().gen_ascii_chars().map(|ch| ch.to_lowercase()).take(len).collect() + rand::thread_rng().gen_ascii_chars().flat_map(|ch| ch.to_lowercase()).take(len).collect() } -fn is_error_eof(err: &io::Error) -> bool { - err.kind() == io::ErrorKind::EndOfFile -} diff --git a/multipart/src/server.rs b/multipart/src/server.rs deleted file mode 100644 index 7e8cd4982..000000000 --- a/multipart/src/server.rs +++ /dev/null @@ -1,480 +0,0 @@ -//! The server-side implementation of `multipart/form-data` requests. -//! -//! Use this when you are implementing a server on top of Hyper and want to -//! to parse and serve POST `multipart/form-data` requests. -//! -//! See the `Multipart` struct for more info. - -use hyper::header::ContentType; -use hyper::server::request::Request; -use hyper::method::Method; - -use mime::{Mime, TopLevel, SubLevel, Attr, Value}; - -use super::{MultipartField, MultipartFile}; - -use std::cmp; -use std::collections::HashMap; -use std::ops::Deref; - -use std::io; -use std::io::prelude::*; - -use std::path::{Path, PathBuf}; - -fn is_multipart_formdata(req: &Request) -> bool { - req.method == Method::Post && req.headers.get::().map_or(false, |ct| { - let ContentType(ref mime) = *ct; - - debug!("Content-Type: {}", mime); - - match *mime { - Mime(TopLevel::Multipart, SubLevel::FormData, _) => true, - _ => false, - } - }) -} - -fn get_boundary(ct: &ContentType) -> Option { - let ContentType(ref mime) = *ct; - let Mime(_, _, ref params) = *mime; - - params.iter().find(|&&(ref name, _)| - if let Attr::Ext(ref name) = *name { - "boundary" == &**name - } else { false } - ).and_then(|&(_, ref val)| - if let Value::Ext(ref val) = *val { - Some(val.clone()) - } else { None } - ) -} - -/// The server-side implementation of `multipart/form-data` requests. -/// -/// Create this with `Multipart::from_request()` passing a `server::Request` object from Hyper, -/// or give Hyper a `handler::Switch` instance instead, -/// then read individual entries with `.read_entry()` or process them all at once with -/// `.foreach_entry()`. -/// -/// Implements `Deref` to allow access to read-only fields on `Request` without copying. -pub struct Multipart<'a> { - source: BoundaryReader>, - tmp_dir: String, -} - -macro_rules! try_find( - ($haystack:expr, $f:ident, $needle:expr, $err:expr, $line:expr) => ( - try!($haystack.$f($needle).ok_or(line_error($err, $line.clone()))) - ) -); - -impl<'a> Multipart<'a> { - - /// If the given `Request` is a POST request of `Content-Type: multipart/form-data`, - /// return the wrapped request as `Ok(Multipart)`, otherwise `Err(Request)`. - /// - /// See the `handler` submodule for a convenient wrapper for this function, - /// `Switch`, that implements `hyper::server::Handler`. - pub fn from_request(req: Request<'a>) -> Result, Request<'a>> { - if !is_multipart_formdata(&req) { return Err(req); } - - let boundary = if let Some(boundary) = req.headers.get::() - .and_then(get_boundary) { boundary } else { return Err(req); }; - - debug!("Boundary: {}", boundary); - - Ok(Multipart { source: BoundaryReader::from_reader(req, format!("--{}\r\n", boundary)), tmp_dir: ::random_alphanumeric(10) }) - } - - /// Read an entry from this multipart request, returning a pair with the field's name and - /// contents. This will return an End of File error if there are no more entries. - /// - /// To get to the data, you will need to match on `MultipartField`. - /// - /// ##Warning - /// If the last returned entry had contents of type `MultipartField::File`, - /// calling this again will discard any unread contents of that entry! - pub fn read_entry<'b>(&'b mut self) -> io::Result<(String, MultipartField<'b>)> { - try!(self.source.consume_boundary()); - let (disp_type, field_name, filename) = try!(self.read_content_disposition()); - - if disp_type != "form-data" { - return Err(io::Error::new( - io::ErrorKind::OtherIoError, - format!("Content-Disposition value: {:?} expected: \"form-data\"", disp_type), - )); - } - - if let Some(content_type) = try!(self.read_content_type()) { - let _ = try!(self.source.read_line()); // Consume empty line - Ok((field_name, - MultipartField::File( - MultipartFile::from_octet(filename, &mut self.source, &content_type, &self.tmp_dir) - ) - )) - } else { - // Empty line consumed by read_content_type() - let text = try!(self.source.read_to_string()); - // The last two characters are "\r\n". - // We can't do a simple trim because the content might be terminated - // with line separators we want to preserve. - Ok((field_name, MultipartField::Text(text[..text.len() - 2].into_string()))) - } - } - - /// Call `f` for each entry in the multipart request. - /// This is a substitute for `Multipart` implementing `Iterator`, - /// since `Iterator::next()` can't use bound lifetimes. - /// - /// See https://www.reddit.com/r/rust/comments/2lkk\4\isize/concrete_lifetime_vs_bound_lifetime/ - pub fn foreach_entry FnMut(String, MultipartField<'b>)>(&'a mut self, mut f: F) { - loop { - match self.read_entry() { - Ok((name, field)) => f(name, field), - Err(err) => { - if err.kind() != io::ErrorKind::EndOfFile { - error!("Error reading Multipart: {}", err); - } - - break; - }, - } - } - } - - fn read_content_disposition(&mut self) -> io::Result<(String, String, Option)> { - let line = try!(self.source.read_line()); - - // Find the end of CONT_DISP in the line - let disp_type = { - const CONT_DISP: &'static str = "Content-Disposition:"; - - let disp_idx = try_find!(&line, find_str, CONT_DISP, - "Content-Disposition subheader not found!", line) + CONT_DISP.len(); - - let disp_type_end = try_find!(line[disp_idx..], find, ';', - "Error parsing Content-Disposition value!", line); - - line[disp_idx .. disp_idx + disp_type_end].trim().into_string() - }; - - let field_name = { - const NAME: &'static str = "name=\""; - - let name_idx = try_find!(&line, find_str, NAME, - "Error parsing field name!", line) + NAME.len(); - - let name_end = try_find!(line[name_idx ..], find, '"', - "Error parsing field name!", line); - - line[name_idx .. name_idx + name_end].into_string() // No trim here since it's in quotes. - }; - - let filename = { - const FILENAME: &'static str = "filename=\""; - - let filename_idx = line.find_str(FILENAME).map(|idx| idx + FILENAME.len()); - let filename_idxs = with(filename_idx, |&start| line[start ..].find('"')); - - filename_idxs.map(|(start, end)| line[start .. start + end].into_string()) - }; - - Ok((disp_type, field_name, filename)) - } - - fn read_content_type(&mut self) -> io::Result> { - debug!("Read content type!"); - let line = try!(self.source.read_line()); - - const CONTENT_TYPE: &'static str = "Content-Type:"; - - let type_idx = (&*line).find_str(CONTENT_TYPE); - - // FIXME Will not properly parse for multiple files! - // Does not expect boundary= - Ok(type_idx.map(|start| line[start + CONTENT_TYPE.len()..].trim().into_string())) - } - - /// Read the request fully, parsing all fields and saving all files - /// to the given directory (if given) and return the result. - /// - /// If `dir` is none, chooses a random subdirectory under `std::os::tmpdir()`. - pub fn save_all(mut self, dir: Option<&Path>) -> io::Result { - let dir = dir.map_or_else(|| ::std::env::temp_dir().join(&self.tmp_dir), |path| path.clone()); - - let mut entries = Entries::with_path(dir); - - loop { - match self.read_entry() { - Ok((name, MultipartField::Text(text))) => { entries.fields.insert(name, text); }, - Ok((name, MultipartField::File(mut file))) => { - let path = try!(file.save_in(&entries.dir)); - entries.files.insert(name, path); - }, - Err(err) => { - if super::is_error_eof(&err) { - error!("Error reading Multipart: {}", err); - } - - break; - }, - } - } - - Ok(entries) - } -} - -impl<'a> Deref for Multipart<'a> { - type Target = Request<'a>; - fn deref(&self) -> &Request<'a> { - self.source.borrow_reader() - } -} - -fn with Option>(left: Option, right: F) -> Option<(T, U)> { - let temp = left.as_ref().and_then(right); - match (left, temp) { - (Some(lval), Some(rval)) => Some((lval, rval)), - _ => None, - } -} - -fn line_error(msg: &str, line: String) -> io::Error { - io::Error::new( - io::ErrorKind::Other, - format!("Error: {:?} on line of request: {:?}", msg, line) - ) -} - -/// A result of `Multipart::save_all()`. -pub struct Entries { - pub fields: HashMap, - pub files: HashMap, - /// The directory the files were saved under. - pub dir: Path, -} - -impl Entries { - fn with_path(path: Path) -> Entries { - Entries { - fields: HashMap::new(), - files: HashMap::new(), - dir: path, - } - } -} - -/* FIXME: Can't have an iterator return a borrowed reference -impl<'a> Iterator<(String, MultipartField<'a>)> for Multipart<'a> { - fn next(&mut self) -> Option<(String, MultipartField<'a>)> { - match self.read_entry() { - Ok(ok) => Some(ok), - Err(err) => { - if err.kind != EndOfFile { - error!("Error reading Multipart: {}", err); - } - - None - }, - } - } -} -*/ - -/// A struct implementing `Read` that will yield bytes until it sees a given sequence. -pub struct BoundaryReader { - reader: S, - boundary: Vec, - last_search_idx: usize, - boundary_read: bool, - buf: Vec, - buf_len: usize, -} - -fn eof() -> io::Result { - Err(io::Error::new(io::ErrorKind::EndOfFile, "End of the stream was reached!")) -} - -const BUF_SIZE: usize = 1024 * 64; // 64k buffer - -impl BoundaryReader where S: Read { - fn from_reader(reader: S, boundary: String) -> BoundaryReader { - let mut buf = Vec::with_capacity(BUF_SIZE); - unsafe { buf.set_len(BUF_SIZE); } - - BoundaryReader { - reader: reader, - boundary: boundary.into_bytes(), - last_search_idx: 0, - boundary_read: false, - buf: buf, - buf_len: 0, - } - } - - fn read_to_boundary(&mut self) -> io::Result<()> { - if !self.boundary_read { - try!(self.true_fill_buf()); - - if self.buf_len == 0 { return eof(); } - - let lookahead = &self.buf[self.last_search_idx .. self.buf_len]; - - let search_idx = lookahead.position_elem(&self.boundary[0]) - .unwrap_or(lookahead.len() - 1); - - debug!("Search idx: {}", search_idx); - - self.boundary_read = lookahead[search_idx..] - .starts_with(&self.boundary); - - self.last_search_idx += search_idx; - - if !self.boundary_read { - self.last_search_idx += 1; - } - - Ok(()) - } else if self.last_search_idx == 0 { - eof() - } - } - - /// Read bytes until the reader is full - fn true_fill_buf(&mut self) -> io::Result<()> { - let mut bytes_read = 1usize; - - while bytes_read != 0 { - bytes_read = match self.reader.read(&mut self.buf[self.buf_len..]) { - Ok(read) => read, - Err(err) => if super::is_error_eof(&err) { break; } else { return Err(err); }, - }; - - self.buf_len += bytes_read; - } - - Ok(()) - } - - fn _consume(&mut self, amt: usize) { - use std::ptr; - - assert!(amt <= self.buf_len); - - let src = self.buf[amt..].as_ptr(); - let dest = self.buf.as_mut_ptr(); - - unsafe { ptr::copy(dest, src, self.buf_len - amt); } - - self.buf_len -= amt; - self.last_search_idx -= amt; - } - - fn consume_boundary(&mut self) -> io::Result<()> { - while !self.boundary_read { - match self.read_to_boundary() { - Ok(_) => (), - Err(err) => if super::is_error_eof(&err) { - break; - } else { - return Err(err); - } - } - } - - let consume_amt = cmp::min(self.buf_len, self.last_search_idx + self.boundary.len()); - - debug!("Consume amt: {} Buf len: {}", consume_amt, self.buf_len); - - self._consume(consume_amt); - self.last_search_idx = 0; - self.boundary_read = false; - - Ok(()) - } - - #[allow(unused)] - fn set_boundary(&mut self, boundary: String) { - self.boundary = boundary.into_bytes(); - } - - pub fn borrow_reader<'a>(&'a self) -> &'a S { - &self.reader - } -} - -impl Read for BoundaryReader where S: Read { - fn read(&mut self, buf: &mut [u8]) -> io::Result { - use std::cmp; - use std::slice::bytes::copy_memory; - - try!(self.read_to_boundary()); - - let trunc_len = cmp::min(buf.len(), self.last_search_idx); - copy_memory(buf, &self.buf[..trunc_len]); - - self._consume(trunc_len); - - Ok(trunc_len) - } -} - -impl BufRead for BoundaryReader where S: Read { - fn fill_buf(&mut self) -> io::Result<&[u8]> { - try!(self.read_to_boundary()); - - let buf = &self.buf[..self.last_search_idx]; - - Ok(buf) - } - - fn consume(&mut self, amt: usize) { - assert!(amt <= self.last_search_idx); - self._consume(amt); - } -} - -pub trait HttpRequest: Read { - -} - - -#[test] -fn test_boundary() { - use std::io::BufReader; - - const BOUNDARY: &'static str = "--boundary\r\n"; - const TEST_VAL: &'static str = "\r ---boundary\r -dashed-value-1\r ---boundary\r -dashed-value-2\r ---boundary\r -"; - - let test_reader = BufReader::new(TEST_VAL.as_bytes()); - let mut reader = BoundaryReader::from_reader(test_reader, BOUNDARY.into_string()); - - debug!("Read 1"); - let string = reader.read_to_string().unwrap(); - debug!("{}", string); - assert!(string.trim().is_empty()); - - debug!("Consume 1"); - reader.consume_boundary().unwrap(); - - debug!("Read 2"); - assert_eq!(reader.read_to_string().unwrap().trim(), "dashed-value-1"); - - debug!("Consume 2"); - reader.consume_boundary().unwrap(); - - debug!("Read 3"); - assert_eq!(reader.read_to_string().unwrap().trim(), "dashed-value-2"); - - debug!("Consume 3"); - reader.consume_boundary().unwrap(); - -} diff --git a/multipart/src/server/boundary.rs b/multipart/src/server/boundary.rs new file mode 100644 index 000000000..be5c8506e --- /dev/null +++ b/multipart/src/server/boundary.rs @@ -0,0 +1,195 @@ +use std::cmp; +use std::borrow::Borrow; + +use std::io; +use std::io::prelude::*; + +use std::ptr; + +/// A struct implementing `Read` that will yield bytes until it sees a given sequence. +pub struct BoundaryReader { + reader: S, + boundary: Vec, + last_search_idx: usize, + boundary_read: bool, + buf: Vec, + buf_len: usize, +} + +const BUF_SIZE: usize = 1024 * 64; // 64k buffer + +impl BoundaryReader where S: Read { + fn from_reader(reader: S, boundary: String) -> BoundaryReader { + let mut buf = vec![0u8; BUF_SIZE]; + + BoundaryReader { + reader: reader, + boundary: boundary.into_bytes(), + last_search_idx: 0, + boundary_read: false, + buf: buf, + buf_len: 0, + } + } + + fn read_to_boundary(&mut self) -> io::Result<()> { + if !self.boundary_read { + try!(self.true_fill_buf()); + + if self.buf_len == 0 { return Ok(()); } + + let lookahead = &self.buf[self.last_search_idx .. self.buf_len]; + + let search_idx = lookahead.position_elem(&self.boundary[0]) + .unwrap_or(lookahead.len() - 1); + + debug!("Search idx: {}", search_idx); + + self.boundary_read = lookahead[search_idx..] + .starts_with(&self.boundary); + + self.last_search_idx += search_idx; + + if !self.boundary_read { + self.last_search_idx += 1; + } + } + + Ok(()) + } + + /// Read bytes until the reader is full + fn true_fill_buf(&mut self) -> io::Result<()> { + let mut bytes_read = 0; + + loop { + bytes_read = try!(self.reader.read(&mut self.buf[self.buf_len..])); + if bytes_read == 0 { break; } + self.buf_len += bytes_read; + } + + Ok(()) + } + + fn _consume(&mut self, amt: usize) { + use std::ptr; + + assert!(amt <= self.buf_len); + + let (dest, src) = self.buf.split_at_mut(amt); + + copy_bytes(src, dest); + + self.buf_len -= amt; + self.last_search_idx -= amt; + } + + fn consume_boundary(&mut self) -> io::Result<()> { + while !self.boundary_read { + try!(self.read_to_boundary()); + } + + let consume_amt = cmp::min(self.buf_len, self.last_search_idx + self.boundary.len()); + + debug!("Consume amt: {} Buf len: {}", consume_amt, self.buf_len); + + self._consume(consume_amt); + self.last_search_idx = 0; + self.boundary_read = false; + + Ok(()) + } + + #[allow(unused)] + fn set_boundary(&mut self, boundary: String) { + self.boundary = boundary.into_bytes(); + } +} + +impl Borrow for BoundaryReader { + fn borrow(&self) -> &R { + &self.reader + } +} + +impl Read for BoundaryReader where R: Read { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + use std::cmp; + + try!(self.read_to_boundary()); + + let trunc_len = cmp::min(buf.len(), self.last_search_idx); + copy_bytes(&self.buf[..trunc_len], buf); + + self._consume(trunc_len); + + Ok(trunc_len) + } +} + +impl BufRead for BoundaryReader where R: Read { + fn fill_buf(&mut self) -> io::Result<&[u8]> { + try!(self.read_to_boundary()); + + let buf = &self.buf[..self.last_search_idx]; + + Ok(buf) + } + + fn consume(&mut self, amt: usize) { + assert!(amt <= self.last_search_idx); + self._consume(amt); + } +} + +// copied from `std::slice::bytes` due to unstable +fn copy_bytes(src: &[u8], dst: &mut [u8]) { + let len_src = src.len(); + assert!(dst.len() >= len_src); + // `dst` is unaliasable, so we know statically it doesn't overlap with `src`. + unsafe { + ptr::copy_nonoverlapping( + src.as_ptr(), + dst.as_mut_ptr(), + len_src + ); + } +} + +#[test] +fn test_boundary() { + use std::io::BufReader; + + const BOUNDARY: &'static str = "--boundary\r\n"; + const TEST_VAL: &'static str = "\r +--boundary\r +dashed-value-1\r +--boundary\r +dashed-value-2\r +--boundary\r +"; + + let test_reader = BufReader::new(TEST_VAL.as_bytes()); + let mut reader = BoundaryReader::from_reader(test_reader, BOUNDARY.to_owned()); + + debug!("Read 1"); + let string = reader.read_to_string().unwrap(); + debug!("{}", string); + assert!(string.trim().is_empty()); + + debug!("Consume 1"); + reader.consume_boundary().unwrap(); + + debug!("Read 2"); + assert_eq!(reader.read_to_string().unwrap().trim(), "dashed-value-1"); + + debug!("Consume 2"); + reader.consume_boundary().unwrap(); + + debug!("Read 3"); + assert_eq!(reader.read_to_string().unwrap().trim(), "dashed-value-2"); + + debug!("Consume 3"); + reader.consume_boundary().unwrap(); + +} diff --git a/multipart/src/server/handler.rs b/multipart/src/server/hyper.rs similarity index 75% rename from multipart/src/server/handler.rs rename to multipart/src/server/hyper.rs index 30434de9b..a0fbe43cb 100644 --- a/multipart/src/server/handler.rs +++ b/multipart/src/server/hyper.rs @@ -1,5 +1,7 @@ //! Convenient wrappers for `hyper::server::Handler` +use hyper::header::ContentType; +use hyper::method::Method; use hyper::server::{Handler, Request, Response}; use super::Multipart; @@ -70,3 +72,32 @@ impl Handler for UnboxedHandler where F: Fn(Request, Response) + Send + Sy (self.f)(req, res); } } + +fn is_multipart_formdata(req: &Request) -> bool { + req.method == Method::Post && req.headers.get::().map_or(false, |ct| { + let ContentType(ref mime) = *ct; + + debug!("Content-Type: {}", mime); + + match *mime { + Mime(TopLevel::Multipart, SubLevel::FormData, _) => true, + _ => false, + } + }) +} + +fn get_boundary(ct: &ContentType) -> Option { + let ContentType(ref mime) = *ct; + let Mime(_, _, ref params) = *mime; + + params.iter().find(|&&(ref name, _)| + if let Attr::Ext(ref name) = *name { + "boundary" == &**name + } else { false } + ).and_then(|&(_, ref val)| + if let Value::Ext(ref val) = *val { + Some(val.clone()) + } else { None } + ) +} + diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs new file mode 100644 index 000000000..699f1d019 --- /dev/null +++ b/multipart/src/server/mod.rs @@ -0,0 +1,274 @@ +//! The server-side implementation of `multipart/form-data` requests. +//! +//! Use this when you are implementing a server on top of Hyper and want to +//! to parse and serve POST `multipart/form-data` requests. +//! +//! See the `Multipart` struct for more info. + +use mime::{Mime, TopLevel, SubLevel, Attr, Value}; + +use super::{MultipartField, MultipartFile}; + +use std::borrow::Borrow; +use std::cmp; +use std::collections::HashMap; +use std::ops::Deref; + +use std::io; +use std::io::prelude::*; + +use std::path::{Path, PathBuf}; + +#[doc(inline)] +pub use self::boundary::BoundaryReader; + +mod boundary; + +#[cfg(feature = "hyper")] +pub mod hyper; + +/// The server-side implementation of `multipart/form-data` requests. +/// +/// Create this with `Multipart::from_request()` passing a `server::Request` object from Hyper, +/// or give Hyper a `handler::Switch` instance instead, +/// then read individual entries with `.read_entry()` or process them all at once with +/// `.foreach_entry()`. +/// +/// Implements `Deref` to allow access to read-only fields on `Request` without copying. +pub struct Multipart { + source: BoundaryReader, + line_buf: String, +} + +macro_rules! try_find( + ($needle:expr, $haystack:expr, $err:expr) => ( + try!($haystack.find($needle).ok_or_else(|| line_error($err, $haystack))) + ) +); + +impl Multipart where R: HttpRequest { + /// If the given `HttpRequest` is a POST request of `Content-Type: multipart/form-data`, + /// return the wrapped request as `Ok(Multipart)`, otherwise `Err(HttpRequest)`. + pub fn from_request(req: R) -> Result, R> { + if !req.is_multipart() { return Err(req); } + + let boundary = match req.get_boundary() { + Some(boundary) => format!("{}\r\n", boundary), + None => return Err(req), + }; + + debug!("Boundary: {}", boundary); + + Ok( + Multipart { + source: BoundaryReader::from_reader(req, boundary), + line_buf: String::new(), + } + ) + } + + /// Read an entry from this multipart request, returning a pair with the field's name and + /// contents. This will return an End of File error if there are no more entries. + /// + /// To get to the data, you will need to match on `MultipartField`. + /// + /// ##Warning + /// If the last returned entry had contents of type `MultipartField::File`, + /// calling this again will discard any unread contents of that entry! + pub fn read_entry(&mut self) -> io::Result<(String, MultipartField)> { + try!(self.source.consume_boundary()); + let (disp_type, field_name, filename) = try!(self.read_content_disposition()); + + if disp_type != "form-data" { + return Err(io::Error::new( + io::ErrorKind::Other, + format!("Content-Disposition value: {:?} expected: \"form-data\"", disp_type), + )); + } + + if let Some(content_type) = try!(self.read_content_type()) { + let _ = try!(self.read_line()); // Consume empty line + Ok((field_name, + MultipartField::File( + MultipartFile::from_octet(filename, &mut self.source, &content_type) + ) + )) + } else { + // Empty line consumed by read_content_type() + let text = try!(self.read_to_string()); + // The last two characters are "\r\n". + // We can't do a simple trim because the content might be terminated + // with line separators we want to preserve. + Ok((field_name, MultipartField::Text(text[..text.len() - 2].into()))) + } + } + + /// Call `f` for each entry in the multipart request. + /// This is a substitute for `Multipart` implementing `Iterator`, + /// since `Iterator::next()` can't use bound lifetimes. + /// + /// See https://www.reddit.com/r/rust/comments/2lkk\4\isize/concrete_lifetime_vs_bound_lifetime/ + pub fn foreach_entry(&mut self, mut foreach: F) where F: FnMut(String, MultipartField) { + loop { + match self.read_entry() { + Ok((name, field)) => foreach(name, field), + Err(err) => { + error!("Error reading Multipart: {}", err); + break; + }, + } + } + } + + fn read_content_disposition(&mut self) -> io::Result<(String, String, Option)> { + let line = try!(self.read_line()); + + // Find the end of CONT_DISP in the line + let disp_type = { + const CONT_DISP: &'static str = "Content-Disposition:"; + + let disp_idx = try_find!(CONT_DISP, &line, "Content-Disposition subheader not found!") + + CONT_DISP.len(); + + let disp_type_end = try_find!( + ';', &line[disp_idx..], + "Error parsing Content-Disposition value!" + ); + + line[disp_idx .. disp_idx + disp_type_end].trim().to_owned() + }; + + let field_name = { + const NAME: &'static str = "name=\""; + + let name_idx = try_find!(NAME, &line, "Error parsing field name!") + NAME.len(); + let name_end = try_find!('"', &line[name_idx ..], "Error parsing field name!"); + + line[name_idx .. name_idx + name_end].to_owned() // No trim here since it's in quotes. + }; + + let filename = { + const FILENAME: &'static str = "filename=\""; + + let filename_idx = line.find(FILENAME).map(|idx| idx + FILENAME.len()); + let filename_idxs = with(filename_idx, |&start| line[start ..].find('"')); + + filename_idxs.map(|(start, end)| line[start .. start + end].to_owned()) + }; + + Ok((disp_type, field_name, filename)) + } + + fn read_content_type(&mut self) -> io::Result> { + debug!("Read content type!"); + let line = try!(self.read_line()); + + const CONTENT_TYPE: &'static str = "Content-Type:"; + + let type_idx = line.find(CONTENT_TYPE); + + // FIXME Will not properly parse for multiple files! + // Does not expect boundary= + Ok(type_idx.map(|start| line[(start + CONTENT_TYPE.len())..].trim().to_owned())) + } + + /// Read the request fully, parsing all fields and saving all files + /// to the given directory (if given) and return the result. + /// + /// If `dir` is none, chooses a random subdirectory under `std::os::tmpdir()`. + pub fn save_all(mut self, dir: Option<&Path>) -> io::Result { + let tmp_dir = super::random_alphanumeric(12); + let dir = dir.map_or_else(|| ::std::env::temp_dir().join(tmp_dir), |path| path.to_owned()); + + let mut entries = Entries::with_path(dir); + + loop { + match self.read_entry() { + Ok((name, MultipartField::Text(text))) => { entries.fields.insert(name, text.into_owned()); }, + Ok((name, MultipartField::File(mut file))) => { + let path = try!(file.save_in(&entries.dir)); + entries.files.insert(name, path); + }, + Err(err) => { + error!("Error reading Multipart: {}", err); + break; + }, + } + } + + Ok(entries) + } + + fn read_line(&mut self) -> io::Result<&str> { + self.source.read_line(&mut self.line_buf).map(|read| &self.line_buf[..read]) + } + + fn read_to_string(&mut self) -> io::Result<&str> { + self.source.read_to_string(&mut self.line_buf).map(|read| &self.line_buf[..read]) + } +} + +impl Deref for Multipart where R: HttpRequest { + type Target = R; + fn deref(&self) -> &R { + self.source.borrow() + } +} + +fn with Option>(left: Option, right: F) -> Option<(T, U)> { + let temp = left.as_ref().and_then(right); + match (left, temp) { + (Some(lval), Some(rval)) => Some((lval, rval)), + _ => None, + } +} + +fn line_error(msg: &str, line: &str) -> io::Error { + io::Error::new( + io::ErrorKind::Other, + format!("Error: {:?} on line of request: {:?}", msg, line) + ) +} + +/// A result of `Multipart::save_all()`. +pub struct Entries { + pub fields: HashMap, + pub files: HashMap, + /// The directory the files were saved under. + pub dir: PathBuf, +} + +impl Entries { + fn with_path>(path: P) -> Entries { + Entries { + fields: HashMap::new(), + files: HashMap::new(), + dir: path.into(), + } + } +} + +/* FIXME: Can't have an iterator return a borrowed reference +impl<'a> Iterator<(String, MultipartField<'a>)> for Multipart<'a> { + fn next(&mut self) -> Option<(String, MultipartField<'a>)> { + match self.read_entry() { + Ok(ok) => Some(ok), + Err(err) => { + if err.kind != EndOfFile { + error!("Error reading Multipart: {}", err); + } + + None + }, + } + } +} +*/ + +pub trait HttpRequest: Read { + fn is_multipart(&self) -> bool; + fn get_boundary(&self) -> Option<&str>; +} + + + From 0a70046e1c707577f54c45fc1d1b053927b051c0 Mon Sep 17 00:00:00 2001 From: Lukas Kalbertodt Date: Sun, 5 Jul 2015 20:32:08 +0200 Subject: [PATCH 051/453] Fix Travis build script (travis has built in Rust support) --- multipart/.travis.yml | 23 +++++++++++------------ multipart/deploy-docs.sh | 6 ++++++ 2 files changed, 17 insertions(+), 12 deletions(-) create mode 100644 multipart/deploy-docs.sh diff --git a/multipart/.travis.yml b/multipart/.travis.yml index 68e9d699d..817238672 100644 --- a/multipart/.travis.yml +++ b/multipart/.travis.yml @@ -1,18 +1,17 @@ language: rust +rust: + - stable + - beta + - nightly os: -- linux -- osx + - linux + - osx env: global: - secure: eJpqXoY/xuCQRYmLBK9dLLzTGHvsBWe67wRN0fljesRgSnU+H09OsWI22DB9z/e6pMKjZpfDuyRkJaHVj8N7rwEfSeKdywv6uKqs5/Mi7dVINnzVA7jU3E4kLc+EuASF8w5d85kfccGNGs7qM0Uwz/i4da3Gw4B+cSc5cqjWsIY= -before_script: -- rustc -v -- cargo -V -script: -- cargo build -v -- cargo test -v -- cargo doc -v --no-deps after_success: -- cp -R target/doc doc -- curl http://www.rust-ci.org/artifacts/put?t=$RUSTCI_TOKEN | sh -- rm -r doc + - | + test ${TRAVIS_PULL_REQUEST} == "false" && \ + test ${TRAVIS_BRANCH} == "master" && \ + test "${TRAVIS_BUILD_NUMBER}.1" == "${TRAVIS_JOB_NUMBER}" && \ + bash deploy-docs.sh diff --git a/multipart/deploy-docs.sh b/multipart/deploy-docs.sh new file mode 100644 index 000000000..13ce69f83 --- /dev/null +++ b/multipart/deploy-docs.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +cargo doc -v --no-deps +cp -R target/doc doc +curl http://www.rust-ci.org/artifacts/put?t=$RUSTCI_TOKEN | sh +rm -r doc From f778cc18aaa1b724308de14bb0f72be0b054a402 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Mon, 6 Jul 2015 16:31:49 -0700 Subject: [PATCH 052/453] WIP refactored client API --- multipart/src/client.rs | 302 ++++++++++++++++++++++------------------ multipart/src/lib.rs | 33 ++--- 2 files changed, 185 insertions(+), 150 deletions(-) diff --git a/multipart/src/client.rs b/multipart/src/client.rs index b8ab6aee0..f97649acf 100644 --- a/multipart/src/client.rs +++ b/multipart/src/client.rs @@ -2,175 +2,205 @@ //! //! Use this when sending POST requests with files to a server. //! -//! See the `Multipart` struct for more info. +//! `ChunkedMultipart` sends chunked requests (recommended), +//! while `SizedMultipart` sends sized requests. +//! +//! Both implement the `MultipartRequest` trait for their core API. +//! +//! Sized requests are more human-readable and use less bandwidth +//! (as chunking adds [significant visual noise and overhead][chunked-example]), +//! but they must be able to load their entirety, including the contents of all files +//! and streams, into memory so the request body can be measured and its size set +//! in the `Content-Length` header. +//! +//! You should really only use sized requests if you intend to inspect the data manually on the +//! server side, as it will produce a more human-readable request body. Also, of course, if the +//! server doesn't support chunked requests or otherwise rejects them. +//! +//! [chunked-example]: http://en.wikipedia.org/wiki/Chunked_transfer_encoding#Example use mime::{Mime, TopLevel, SubLevel, Attr, Value}; -use mime_guess::{guess_mime_type, octet_stream}; - -use std::borrow::Cow; +use std::borrow::{Borrow, BorrowMut}; +use std::cell::Cell; use std::fs::File; use std::io; use std::io::prelude::*; -use super::{MultipartField, MultipartFile, ref_copy, random_alphanumeric}; - -const BOUNDARY_LEN: usize = 8; +use std::marker::PhantomData; -type Fields<'a> = Vec<(Cow<'a, str>, MultipartField<'a>)>; +pub type Multipart = ChunkedMultipart; -/// The entry point of the client-side multipart API. +/// The core API for all multipart requests. /// -/// Add text fields with `.add_text()` and files with `.add_file()`, -/// then obtain a `hyper::client::Request` object and pass it to `.send()`. -pub struct Multipart<'a> { - fields: Fields<'a>, - boundary: String, - /// If the request can be sized. - /// If true, avoid using chunked requests. - /// Defaults to `false`. - pub sized: bool, -} +/// As part of the API contract, all errors in writing to the HTTP stream are stashed until the +/// implementor's concrete `.send()` method is called, or if the last error is inspected with +/// `.last_err()`. +/// +/// However, no reading or writing is performed after an error occurs, to avoid wasting CPU cycles +/// on a request that will not complete, with the exception of API calls after the last error is +/// cleared with `.take_err()`. +pub trait MultipartRequest { + #[doc(hidden)] + type Stream: Write; -impl<'a> Multipart<'a> { + type Error: From; - /// Create a new `Multipart` instance with an empty set of fields. - pub fn new() -> Multipart<'a> { - Multipart { - fields: Vec::new(), - boundary: random_alphanumeric(BOUNDARY_LEN), - sized: false, - } - } + #[doc(hidden)] + fn stream_mut(&mut self) -> &mut Stream; + #[doc(hidden)] + fn write_boundary(&mut self) -> io::Result<()>; + + /// Get a reference to the last error returned from writing to the HTTP stream, if any. + fn last_err(&self) -> Option<&Self::Error>; - /// Add a text field to this multipart request. + /// Remove and return the last error to occur, allowing subsequent API calls to proceed + /// normally. + fn take_err(&mut self) -> Option; + + /// Write a text field to this multipart request. /// `name` and `val` can be either owned `String` or `&str`. - /// Prefer `String` if you're trying to limit allocations and copies. - pub fn add_text>, V: Into>>(&mut self, name: N, val: V) { - self.add_field(name, MultipartField::Text(val.into())); + fn write_text, V: Borrow>(mut self, name: N, val: V) -> Self { + if self.last_err.is_none() { + self.last_err = chain_result! { + self.write_field_headers(name, None, None), + self.write_line(val.borrow()), + self.write_boundary() + }.err().map(E::from) + } + + self } - /// Add the file to the multipart request, guessing its `Content-Type` + /// Write a file to the multipart request, guessing its `Content-Type` /// from its extension and supplying its filename. /// - /// See `add_stream()`. - pub fn add_file>>(&mut self, name: N, file: &'a mut File) { - let filename = file.path().filename_str().map(|s| s.to_owned()); - let content_type = guess_mime_type(file.path()); - - self.add_field( - name, - MultipartField::File(MultipartFile::from_file(filename, file, content_type)) - ); + /// See `write_stream()` for more info. + fn write_file, F: BorrowMut>(&mut self, name: N, file: F) -> Self { + if self.last_err.is_none() { + self.last_err = chain_result! { + { // New borrow scope so we can reborrow `file` after + let file_path = file.borrow().path(); + let content_type = mime_guess::guess_mime_type(file_path); + self.write_field_headers(name.borrow(), file_path.filename_str(), content_type) + }, + io::copy(file.borrow_mut(), self.stream_mut()), + self.write_boundary() + }; + } + + self } - /// Add a `Read` as a file field, supplying `filename` if given, + /// Write a byte stream to the multipart request as a file field, supplying `filename` if given, /// and `content_type` if given or `application/octet-stream` if not. /// /// ##Warning - /// The given `Read` **must** be able to read to EOF (end of file/no more data). - /// If it never returns EOF it will be read to infinity (even if it reads 0 bytes forever) - /// and the request will never be completed. + /// The given `Read` **must** be able to read to EOF (end of file/no more data), meaning + /// `Read::read()` returns `Ok(0)`. + /// If it never returns EOF it will be read to infinity and the request will never be completed. /// - /// If `sized` is `true`, this adds an additional consequence of out-of-control - /// memory usage, as `Multipart` tries to read an infinite amount of data into memory. + /// In the case of `SizedMultipart` this also can cause out-of-control memory usage as the + /// multipart data has to be written to an in-memory buffer so it can be measured. /// - /// Use `std::io::util::LimitReader` if you wish to send data from a `Read` - /// that will never return EOF otherwise. - pub fn add_stream>>(&mut self, name: N, reader: &'a mut (Read + 'a), - filename: Option, content_type: Option) { - self.add_field(name, - MultipartField::File(MultipartFile { - filename: filename, - content_type: content_type.unwrap_or_else(octet_stream), - reader: reader, - tmp_dir: None, - }) - ); - } - - fn add_field>>(&mut self, name: N, val: MultipartField<'a>) { - self.fields.push((name.into_string(), val)); - } - - /// Apply the appropriate headers to the `Request` (obtained from Hyper) and send the data. - /// If `self.sized == true`, send a sized (non-chunked) request, setting the `Content-Length` - /// header. Else, send a chunked request. - /// - /// Sized requests are more human-readable and use less bandwidth - /// (as chunking adds [significant visual noise and overhead][chunked-example]), - /// but they must be able to load their entirety, including the contents of all files - /// and streams, into memory so the request body can be measured and its size set - /// in the `Content-Length` header. - /// - /// Prefer chunked requests when sending very large or numerous files, - /// or when human-readability or bandwidth aren't an issue. - /// - /// [chunked-example]: http://en.wikipedia.org/wiki/Chunked_transfer_encoding#Example - /// - /// ##Panics - /// If `req` fails sanity checks in `HttpRequest::apply_headers()`. - pub fn send(self, mut req: R) -> R::RequestResult where R: HttpRequest { - debug!("Fields: {:?}; Boundary: {:?}", self.fields, self.boundary); - - if self.sized { - return self.send_sized(req); + /// Use `Read::take` if you wish to send data from a `Read` that will never end otherwise. + fn write_stream, F: Borrow, Rt: Read, R: Borrow>( + mut self, name: N, read: R, filename: Option, content_type: Option + ) -> Self { + if self.last_err.is_none() { + let content_type = content_type.map_or_else(mime_guess::octet_stream); + + self.last_err = chain_result! { + self.write_field_headers(name.borrow(), filename, content_type), + io::copy(read.borrow_mut(), self.stream_mut()), + self.write_boundary() + }; } - let Multipart { fields, boundary, ..} = self; - - req.apply_headers(&boundary, None); + self + } - req.send(|req| write_body(&mut req, fields, &boundary)) + #[doc(hidden)] + fn write_field_headers(&mut self, name: &str, filename: Option<&str>, content_type: Option) + -> io::Result<()> { + chain_result! { + write!(self.stream_mut(), "Content-Disposition: form-data; name=\"{}\"", name), + filename.map(|filename| write!(self.stream_mut(), "; filename=\"{}\"", filename)) + .unwrap_or(Ok(())), + content_type.map(|content_type| write!(self.stream_mut(), "\r\nContent-Type: {}", content_type)) + .unwrap_or(Ok(())), + self.write_line("\r\n") + } } - - fn send_sized(self, mut req: R) -> R::RequestResult where R: HttpRequest { - let mut body: Vec = Vec::new(); - let Multipart { fields, boundary, ..} = self; + #[doc(hidden)] + fn write_line(&mut self, line: &str) -> io::Result<()> { + write!(self.stream_mut(), "{}\r\n", line) + } +} - try!(write_body(&mut body, fields, &boundary)); - - req.apply_headers(&boundary, Some(body.len())); - req.send(|req| req.write(&body)) - } +/// The entry point of the client-side multipart API. +/// +/// Add text fields with `.add_text()` and files with `.add_file()`, +/// then obtain a `hyper::client::Request` object and pass it to `.send()`. +pub struct ChunkedMultipart { + request: PhantomData, + stream: R::Stream, + boundary: String, + last_err: Option, } +impl ChunkedMultipart { + /// Create a new `ChunkedMultipart` to wrap a request. + /// + /// ##May Panic + /// If `req` fails sanity checks in `HttpRequest::apply_headers()`. + pub fn from_request(mut req: R) -> Result, R::Error> { + let boundary = ::gen_boundary(); + req.apply_headers(&boundary, None); -fn write_body(wrt: &mut W, fields: Fields, boundary: &str) -> io::Result<()> { - try!(write_boundary(wrt, boundary)); + let stream = try!(req.open_stream()); - for (name, field) in fields.into_iter() { - try!(write_field(wrt, name, field, boundary)); + Multipart { + request: PhantomData, + stream: stream, + boundary: boundary, + last_err: None + } } - Ok(()) -} - -fn write_field(wrt: &mut Write, name: String, field: MultipartField, boundary: &str) -> io::Result<()> { - try!(write!(wrt, "Content-Disposition: form-data; name=\"{}\"\r\n\r\n", name)); + /// Finalize the request and return the response from the server. + pub fn send(self) -> R::Stream::Result where R: HttpRequest { + self.stream.finalize() + } +} - try!(match field { - MultipartField::Text(text) => write_line(wrt, &*text), - MultipartField::File(file) => write_file(wrt, file), - }); - - write_boundary(wrt, boundary) -} +impl MultipartRequest for ChunkedMultipart { + type Stream = R::Stream; + type Error = R::Error; -fn write_boundary(wrt: &mut Write, boundary: &str) -> io::Result<()> { - write!(wrt, "--{}\r\n", boundary) + fn stream_mut(&mut self) -> &mut R::Stream { &mut self.stream } + fn write_boundary(&mut self) -> io::Result<()> { + write!(self.stream, "{}\r\n", boundary) + } } -fn write_file(wrt: &mut Write, mut file: MultipartFile) -> io::Result<()> { - try!(file.filename.map(|filename| write!(wrt, "; filename=\"{}\"\r\n", filename)).unwrap_or(Ok(()))); - try!(write!(wrt, "Content-Type: {}\r\n\r\n", file.content_type)); - ref_copy(&mut file.reader, wrt) +/// A struct for sending a sized multipart request. The API varies subtly from `Multipart`. +/// +/// The request data will be written to a `Vec` so its size can be measured and the +/// `Content-Length` header set when the request is sent. +pub struct SizedMultipart { + stream: Vec, + boundary: String, + last_err: io::Error, } -/// Specialized write_line that writes CRLF after a line as per W3C specs -fn write_line(req: &mut Write, s: &str) -> io::Result<()> { - req.write_str(s).and_then(|_| req.write(b"\r\n")) +impl SizedMultipart { + pub fn new() -> SizedMultipart { + SizedMultipart { + multipart_data: Vec::new() + } + } } fn multipart_mime(bound: &str) -> Mime { @@ -181,10 +211,9 @@ fn multipart_mime(bound: &str) -> Mime { } pub trait HttpRequest { - type RequestStream: Write; - type Response; - type RequestErr: From; - type RequestResult = Result; + type Stream: Write; + type Response: Read; + type Error: From; /// Set the `ContentType` header to `multipart/form-data` and supply the `boundary` value. /// @@ -192,10 +221,19 @@ pub trait HttpRequest { fn apply_headers(&mut self, boundary: &str, content_len: Option); /// Open the request stream and invoke the given closure, where the request body will be /// written. After the closure returns, finalize the request and return its result. - fn send(self, send_fn: F) -> Self::RequestResult - where F: FnOnce(&mut Self::RequestStream) -> io::Result<()>; + fn open_stream(self) -> Result; } +pub trait HttpStream: Write { + type Request: HttpRequest; + type SendResult = Result; + + /// Finalize and close the stream, returning the HTTP response object. + fn finish(self) -> SendResult; +} + + + #[cfg(feature = "hyper")] mod hyper_impl { use hyper::client::request::Request; diff --git a/multipart/src/lib.rs b/multipart/src/lib.rs index 7b3f64cf0..de7d9846c 100644 --- a/multipart/src/lib.rs +++ b/multipart/src/lib.rs @@ -18,6 +18,18 @@ use std::io; use std::io::prelude::*; use std::path::{Path, PathBuf}; +macro_rules! try_all { + ($first_expr:expr, $($try_expr:expr),*) => ( + try!($first_expr $(.and_then(|_| $try_expr))*); + ) +} + +macro_rules! chain_result { + ($first_expr:expr, $($try_expr:expr),*) => ( + $first_expr $(.and_then(|_| $try_expr))* + ) +} + pub mod client; pub mod server; @@ -178,25 +190,10 @@ impl<'a> Read for MultipartFile<'a> { } } -/// A copy of `std::io::util::copy` that takes trait references -pub fn ref_copy(r: &mut Read, w: &mut Write) -> io::Result<()> { - let mut buf = [0; 1024 * 64]; - - loop { - let len = try!(r.read(&mut buf)); - - if len == 0 { break; } - - try!(w.write(&buf[..len])); - } +const BOUNDARY_LEN: usize = 16; - Ok(()) -} - -/// Generate a random alphanumeric sequence of length `len` -fn random_alphanumeric(len: usize) -> String { +fn gen_boundary() -> String { use rand::Rng; - rand::thread_rng().gen_ascii_chars().flat_map(|ch| ch.to_lowercase()).take(len).collect() + "--".chars().chain(thread_rng().gen_ascii_chars().take(BOUNDARY_LEN)).collect() } - From 6cc7b38b873921678fbcbf1957d1d85a9de0eb14 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Mon, 6 Jul 2015 23:59:48 -0700 Subject: [PATCH 053/453] WIP even more --- multipart/src/client.rs | 91 ++++++++++++++++----- multipart/src/lib.rs | 157 +----------------------------------- multipart/src/server/mod.rs | 146 ++++++++++++++++++++++++++++----- 3 files changed, 196 insertions(+), 198 deletions(-) diff --git a/multipart/src/client.rs b/multipart/src/client.rs index f97649acf..426a6fe55 100644 --- a/multipart/src/client.rs +++ b/multipart/src/client.rs @@ -29,7 +29,7 @@ use std::io::prelude::*; use std::marker::PhantomData; -pub type Multipart = ChunkedMultipart; +pub type Multipart = ChunkedMultipart; /// The core API for all multipart requests. /// @@ -47,7 +47,7 @@ pub trait MultipartRequest { type Error: From; #[doc(hidden)] - fn stream_mut(&mut self) -> &mut Stream; + fn stream_mut(&mut self) -> &mut Self::Stream; #[doc(hidden)] fn write_boundary(&mut self) -> io::Result<()>; @@ -66,7 +66,7 @@ pub trait MultipartRequest { self.write_field_headers(name, None, None), self.write_line(val.borrow()), self.write_boundary() - }.err().map(E::from) + }.err().map(Self::Error::from) } self @@ -81,12 +81,12 @@ pub trait MultipartRequest { self.last_err = chain_result! { { // New borrow scope so we can reborrow `file` after let file_path = file.borrow().path(); - let content_type = mime_guess::guess_mime_type(file_path); + let content_type = ::mime_guess::guess_mime_type(file_path); self.write_field_headers(name.borrow(), file_path.filename_str(), content_type) }, io::copy(file.borrow_mut(), self.stream_mut()), self.write_boundary() - }; + }.err().map(Self::Error::from); } self @@ -108,13 +108,13 @@ pub trait MultipartRequest { mut self, name: N, read: R, filename: Option, content_type: Option ) -> Self { if self.last_err.is_none() { - let content_type = content_type.map_or_else(mime_guess::octet_stream); + let content_type = content_type.map_or_else(::mime_guess::octet_stream); self.last_err = chain_result! { self.write_field_headers(name.borrow(), filename, content_type), io::copy(read.borrow_mut(), self.stream_mut()), self.write_boundary() - }; + }.err().map(Self::Error::from); } self @@ -170,8 +170,8 @@ impl ChunkedMultipart { } /// Finalize the request and return the response from the server. - pub fn send(self) -> R::Stream::Result where R: HttpRequest { - self.stream.finalize() + pub fn send(self) -> <::Stream as HttpStream>::Result where R: HttpRequest { + self.last_err.and_then(|_| self.stream.finalize()) } } @@ -179,10 +179,21 @@ impl MultipartRequest for ChunkedMultipart { type Stream = R::Stream; type Error = R::Error; - fn stream_mut(&mut self) -> &mut R::Stream { &mut self.stream } + fn stream_mut(&mut self) -> &mut R::Stream { + &mut self.stream + } + fn write_boundary(&mut self) -> io::Result<()> { - write!(self.stream, "{}\r\n", boundary) - } + write!(self.stream, "{}\r\n", self.boundary) + } + + fn last_err(&self) -> Option<&Self::Error> { + self.last_err.as_ref() + } + + fn take_err(&mut self) -> Option { + self.last_err.take() + } } /// A struct for sending a sized multipart request. The API varies subtly from `Multipart`. @@ -190,17 +201,51 @@ impl MultipartRequest for ChunkedMultipart { /// The request data will be written to a `Vec` so its size can be measured and the /// `Content-Length` header set when the request is sent. pub struct SizedMultipart { - stream: Vec, + data: Vec, boundary: String, - last_err: io::Error, + last_err: Option, } impl SizedMultipart { pub fn new() -> SizedMultipart { SizedMultipart { - multipart_data: Vec::new() + data: Vec::new(), + boundary: ::gen_boundary(), + last_err: None, } } + + pub fn send(self, mut req: R) -> <::Stream as HttpStream>::Result + where R: HttpRequest { + let boundary = ::gen_boundary(); + req.apply_headers(&boundary, Some(self.data.len())); + let req = try!(req.open_stream()); + try!(io::copy(&self.data, &mut req)); + req.finish() + } +} + +impl MultipartRequest for SizedMultipart { + type Stream = Vec; + type Error = io::Error; + + #[doc(hidden)] + fn stream_mut(&mut self) -> &mut Vec { + &mut self.data + } + + #[doc(hidden)] + fn write_boundary(&mut self) -> io::Result<()> { + write!(self.stream, "{}\r\n", self.boundary) + } + + fn last_err(&self) -> Option<&Self::Error> { + self.last_err.as_ref() + } + + fn take_err(&mut self) -> Option { + self.last_err.take() + } } fn multipart_mime(bound: &str) -> Mime { @@ -211,7 +256,7 @@ fn multipart_mime(bound: &str) -> Mime { } pub trait HttpRequest { - type Stream: Write; + type Stream: HttpStream; type Response: Read; type Error: From; @@ -226,14 +271,12 @@ pub trait HttpRequest { pub trait HttpStream: Write { type Request: HttpRequest; - type SendResult = Result; + type Result = Result; /// Finalize and close the stream, returning the HTTP response object. - fn finish(self) -> SendResult; + fn finish(self) -> Self::Result; } - - #[cfg(feature = "hyper")] mod hyper_impl { use hyper::client::request::Request; @@ -244,9 +287,9 @@ mod hyper_impl { use std::io; impl super::HttpRequest for Request { - type RequestStream = Request; + type Stream = Request; type Response = Response; - type RequestErr = HyperError; + type Error = HyperError; /// #Panics /// If the `Request` method is not `Method::Post`. @@ -275,5 +318,9 @@ mod hyper_impl { req.send() } } + + impl super::HttpStream for Request { + + } } diff --git a/multipart/src/lib.rs b/multipart/src/lib.rs index de7d9846c..2af4dc256 100644 --- a/multipart/src/lib.rs +++ b/multipart/src/lib.rs @@ -33,167 +33,12 @@ macro_rules! chain_result { pub mod client; pub mod server; -/// A representation of a file in HTTP `multipart/form-data`. -/// -/// This struct has an input "flavor" and an output "flavor". -/// The input "flavor" is used internally by `client::Multipart::add_file()` -/// and is never exposed to the user. -/// -/// The output "flavor" is returned by `server::Multipart::read_entry()` and represents -/// a file entry in the incoming multipart request. -/// -/// Note that in the output "flavor", the file is not yet saved to the system; -/// instead, the struct implements a `Reader` that points -/// to the beginning of the file's contents in the HTTP stream. -/// You can read it to EOF, or use one of the `save_*()` methods here -/// to save it to disk. -pub struct MultipartFile<'a> { - filename: Option, - content_type: Mime, - reader: &'a mut (Read + 'a), -} - -impl<'a> MultipartFile<'a> { - fn from_octet(filename: Option, reader: &'a mut Read, cont_type: &str) -> MultipartFile<'a> { - MultipartFile { - filename: filename, - reader: reader, - content_type: cont_type.parse::().ok().unwrap_or_else(mime_guess::octet_stream), - } - } - - fn from_file(filename: Option, reader: &'a mut File, mime: Mime) -> MultipartFile<'a> { - MultipartFile { - filename: filename, - reader: reader, - content_type: mime, - } - } - - /// Save this file to `path`, discarding the filename. - /// - /// If successful, the file can be found at `path`. - pub fn save_as(&mut self, path: &Path) -> io::Result<()> { - let mut file = try!(File::create(path)); - - ref_copy(self.reader, &mut file) - } - - /// Save this file in the directory described by `dir`, - /// appending `filename` if present, or a random string otherwise. - /// - /// Returns the created file's path on success. - /// - /// ###Panics - /// If `dir` does not represent a directory. - pub fn save_in(&mut self, dir: &Path) -> io::Result { - assert!(dir.is_dir(), "Given path is not a directory!"); - - let path = dir.join(self.dest_filename()); - - try!(self.save_as(&path)); - - Ok(path) - } - - /// Save this file in the OS temp directory, returned from `std::env::temp_dir()`. - /// - /// Returns the created file's path on success. - pub fn save_temp(&mut self) -> io::Result { - use std::env; - - self.save_in(&env::temp_dir()) - } - - fn dest_filename(&self) -> String { - self.filename.as_ref().map_or_else(|| random_alphanumeric(10), |s| s.clone()) - } - - pub fn filename(&self) -> Option<&str> { - self.filename.as_ref().map(String::as_ref) - } - - /// Get the content type of this file. - /// On the client, it is guessed by the file extension. - /// On the server, it is retrieved from the request or assumed to be - /// `application/octet-stream`. - pub fn content_type(&self) -> Mime { - self.content_type.clone() - } -} -impl<'a> fmt::Debug for MultipartFile<'a> { - fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { - write!(fmt, "Filename: {:?} Content-Type: {:?}", self.filename, self.content_type) - } -} - - -/// A field in a `multipart/form-data` request. -/// -/// Like `MultipartFile`, this is used in both the client and server-side implementations, -/// but only exposed to the user on the server. -/// -/// This enum does not include the names of the fields, as those are yielded separately -/// by `server::Multipart::read_entry()`. -#[derive(Debug)] -pub enum MultipartField<'a> { - /// A text field. - Text(Cow<'a, str>), - /// A file field, including the content type and optional filename - /// along with a `Read` implementation for getting the contents. - File(MultipartFile<'a>), - // MultiFiles(Vec), /* TODO: Multiple files */ -} - -impl<'a> MultipartField<'a> { - - /// Borrow this field as a text field, if possible. - pub fn as_text(&self) -> Option<&str> { - match *self { - MultipartField::Text(ref s) => Some(s), - _ => None, - } - } - - /// Take this field as a text field, if possible, - /// returning `self` otherwise. - pub fn to_text(self) -> Result, MultipartField<'a>> { - match self { - MultipartField::Text(s) => Ok(s), - _ => Err(self), - } - } - - /// Borrow this field as a file field, if possible - /// Mutably borrows so the contents can be read. - pub fn as_file(&mut self) -> Option<&mut MultipartFile<'a>> { - match *self { - MultipartField::File(ref mut file) => Some(file), - _ => None, - } - } - - /// Take this field as a file field if possible, - /// returning `self` otherwise. - pub fn to_file(self) -> Result, MultipartField<'a>> { - match self { - MultipartField::File(file) => Ok(file), - _ => Err(self), - } - } -} - -impl<'a> Read for MultipartFile<'a> { - fn read(&mut self, buf: &mut [u8]) -> io::Result{ - self.reader.read(buf) - } -} const BOUNDARY_LEN: usize = 16; fn gen_boundary() -> String { use rand::Rng; - "--".chars().chain(thread_rng().gen_ascii_chars().take(BOUNDARY_LEN)).collect() + "--".chars().chain(rand::thread_rng().gen_ascii_chars().take(BOUNDARY_LEN)).collect() } diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index 699f1d019..3df4a5390 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -7,13 +7,14 @@ use mime::{Mime, TopLevel, SubLevel, Attr, Value}; -use super::{MultipartField, MultipartFile}; - use std::borrow::Borrow; use std::cmp; use std::collections::HashMap; use std::ops::Deref; +use std::fmt; + +use std::fs::File; use std::io; use std::io::prelude::*; @@ -175,10 +176,9 @@ impl Multipart where R: HttpRequest { /// Read the request fully, parsing all fields and saving all files /// to the given directory (if given) and return the result. /// - /// If `dir` is none, chooses a random subdirectory under `std::os::tmpdir()`. + /// If `dir` is none, uses `std::os::tmpdir()`. pub fn save_all(mut self, dir: Option<&Path>) -> io::Result { - let tmp_dir = super::random_alphanumeric(12); - let dir = dir.map_or_else(|| ::std::env::temp_dir().join(tmp_dir), |path| path.to_owned()); + let dir = dir.map_or_else(::std::env::temp_dir, |path| path.to_owned()); let mut entries = Entries::with_path(dir); @@ -248,27 +248,133 @@ impl Entries { } } -/* FIXME: Can't have an iterator return a borrowed reference -impl<'a> Iterator<(String, MultipartField<'a>)> for Multipart<'a> { - fn next(&mut self) -> Option<(String, MultipartField<'a>)> { - match self.read_entry() { - Ok(ok) => Some(ok), - Err(err) => { - if err.kind != EndOfFile { - error!("Error reading Multipart: {}", err); - } +pub trait HttpRequest: Read { + fn is_multipart(&self) -> bool; + fn get_boundary(&self) -> Option<&str>; +} + +/// A representation of a file in HTTP `multipart/form-data`. +/// +/// Note that the file is not yet saved to the system; +/// instead, the struct implements a `Reader` that points +/// to the beginning of the file's contents in the HTTP stream. +/// You can read it to EOF, or use one of the `save_*()` methods here +/// to save it to disk. +pub struct MultipartFile<'a, R> { + filename: Option, + content_type: Mime, + reader: &'a mut BoundaryReader, +} + +impl<'a> MultipartFile<'a> { + fn from_octet(filename: Option, reader: &'a mut Read, cont_type: &str) -> MultipartFile<'a> { + MultipartFile { + filename: filename, + reader: reader, + content_type: cont_type.parse::().ok().unwrap_or_else(::mime_guess::octet_stream), + } + } - None - }, + fn from_file(filename: Option, reader: &'a mut File, mime: Mime) -> MultipartFile<'a> { + MultipartFile { + filename: filename, + reader: reader, + content_type: mime, } } + + /// Save this file to `path`, discarding the filename. + /// + /// If successful, the file can be found at `path`. + pub fn save_as(&mut self, path: &Path) -> io::Result<()> { + let mut file = try!(File::create(path)); + io::copy(self.reader, &mut file) + } + + /// Save this file in the directory described by `dir`, + /// appending `filename` if present, or a random string otherwise. + /// + /// Returns the created file's path on success. + /// + /// ###Panics + /// If `dir` does not represent a directory. + pub fn save_in(&mut self, dir: &Path) -> io::Result { + assert!(dir.is_dir(), "Given path is not a directory!"); + + let path = dir.join(self.dest_filename()); + + try!(self.save_as(&path)); + + Ok(path) + } + + /// Save this file in the OS temp directory, returned from `std::env::temp_dir()`. + /// + /// Returns the created file's path on success. + pub fn save_temp(&mut self) -> io::Result { + use std::env; + + self.save_in(&env::temp_dir()) + } + + pub fn filename(&self) -> Option<&str> { + self.filename.as_ref().map(String::as_ref) + } + + /// Get the content type of this file. + /// On the client, it is guessed by the file extension. + /// On the server, it is retrieved from the request or assumed to be + /// `application/octet-stream`. + pub fn content_type(&self) -> Mime { + self.content_type.clone() + } } -*/ -pub trait HttpRequest: Read { - fn is_multipart(&self) -> bool; - fn get_boundary(&self) -> Option<&str>; +impl<'a> Read for MultipartFile<'a> { + fn read(&mut self, buf: &mut [u8]) -> io::Result{ + self.reader.read(buf) + } +} + + +impl<'a> fmt::Debug for MultipartFile<'a> { + fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + write!(fmt, "Filename: {:?} Content-Type: {:?}", self.filename, self.content_type) + } +} + + +/// A field in a `multipart/form-data` request. +/// +/// This enum does not include the names of the fields, as those are yielded separately +/// by `server::Multipart::read_entry()`. +#[derive(Debug)] +pub enum MultipartField<'a> { + /// A text field. + Text(&'a str), + /// A file field, including the content type and optional filename + /// along with a `Read` implementation for getting the contents. + File(MultipartFile<'a>), + // MultiFiles(Vec), /* TODO: Multiple files */ } +impl<'a> MultipartField<'a> { + /// Borrow this field as a text field, if possible. + pub fn as_text(&self) -> Option<&str> { + match *self { + MultipartField::Text(ref s) => Some(s), + _ => None, + } + } + + /// Borrow this field as a file field, if possible + /// Mutably borrows so the contents can be read. + pub fn as_file(&mut self) -> Option<&mut MultipartFile<'a>> { + match *self { + MultipartField::File(ref mut file) => Some(file), + _ => None, + } + } +} From fd54b8a493dcac4487d94ca10942bcabefdbb83b Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Tue, 7 Jul 2015 12:07:08 -0700 Subject: [PATCH 054/453] Why havent I removed this yet --- multipart/mime_types.json | 563 -------------------------------------- 1 file changed, 563 deletions(-) delete mode 100644 multipart/mime_types.json diff --git a/multipart/mime_types.json b/multipart/mime_types.json deleted file mode 100644 index 1ad9b1254..000000000 --- a/multipart/mime_types.json +++ /dev/null @@ -1,563 +0,0 @@ -{ - "_source":"https://github.com/samuelneff/MimeTypeMap/blob/master/src/MimeTypeMap.cs", - "323": "text/h323", - "3g2": "video/3gpp2", - "3gp": "video/3gpp", - "3gp2": "video/3gpp2", - "3gpp": "video/3gpp", - "7z": "application/x-7z-compressed", - "aa": "audio/audible", - "AAC": "audio/aac", - "aaf": "application/octet-stream", - "aax": "audio/vnd.audible.aax", - "ac3": "audio/ac3", - "aca": "application/octet-stream", - "accda": "application/msaccess.addin", - "accdb": "application/msaccess", - "accdc": "application/msaccess.cab", - "accde": "application/msaccess", - "accdr": "application/msaccess.runtime", - "accdt": "application/msaccess", - "accdw": "application/msaccess.webapplication", - "accft": "application/msaccess.ftemplate", - "acx": "application/internet-property-stream", - "AddIn": "text/xml", - "ade": "application/msaccess", - "adobebridge": "application/x-bridge-url", - "adp": "application/msaccess", - "ADT": "audio/vnd.dlna.adts", - "ADTS": "audio/aac", - "afm": "application/octet-stream", - "ai": "application/postscript", - "aif": "audio/x-aiff", - "aifc": "audio/aiff", - "aiff": "audio/aiff", - "air": "application/vnd.adobe.air-application-installer-package+zip", - "amc": "application/x-mpeg", - "application": "application/x-ms-application", - "art": "image/x-jg", - "asa": "application/xml", - "asax": "application/xml", - "ascx": "application/xml", - "asd": "application/octet-stream", - "asf": "video/x-ms-asf", - "ashx": "application/xml", - "asi": "application/octet-stream", - "asm": "text/plain", - "asmx": "application/xml", - "aspx": "application/xml", - "asr": "video/x-ms-asf", - "asx": "video/x-ms-asf", - "atom": "application/atom+xml", - "au": "audio/basic", - "avi": "video/x-msvideo", - "axs": "application/olescript", - "bas": "text/plain", - "bcpio": "application/x-bcpio", - "bin": "application/octet-stream", - "bmp": "image/bmp", - "c": "text/plain", - "cab": "application/octet-stream", - "caf": "audio/x-caf", - "calx": "application/vnd.ms-office.calx", - "cat": "application/vnd.ms-pki.seccat", - "cc": "text/plain", - "cd": "text/plain", - "cdda": "audio/aiff", - "cdf": "application/x-cdf", - "cer": "application/x-x509-ca-cert", - "chm": "application/octet-stream", - "class": "application/x-java-applet", - "clp": "application/x-msclip", - "cmx": "image/x-cmx", - "cnf": "text/plain", - "cod": "image/cis-cod", - "config": "application/xml", - "contact": "text/x-ms-contact", - "coverage": "application/xml", - "cpio": "application/x-cpio", - "cpp": "text/plain", - "crd": "application/x-mscardfile", - "crl": "application/pkix-crl", - "crt": "application/x-x509-ca-cert", - "cs": "text/plain", - "csdproj": "text/plain", - "csh": "application/x-csh", - "csproj": "text/plain", - "css": "text/css", - "csv": "text/csv", - "cur": "application/octet-stream", - "cxx": "text/plain", - "dat": "application/octet-stream", - "datasource": "application/xml", - "dbproj": "text/plain", - "dcr": "application/x-director", - "def": "text/plain", - "deploy": "application/octet-stream", - "der": "application/x-x509-ca-cert", - "dgml": "application/xml", - "dib": "image/bmp", - "dif": "video/x-dv", - "dir": "application/x-director", - "disco": "text/xml", - "dll": "application/x-msdownload", - "dll.config": "text/xml", - "dlm": "text/dlm", - "doc": "application/msword", - "docm": "application/vnd.ms-word.document.macroEnabled.12", - "docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", - "dot": "application/msword", - "dotm": "application/vnd.ms-word.template.macroEnabled.12", - "dotx": "application/vnd.openxmlformats-officedocument.wordprocessingml.template", - "dsp": "application/octet-stream", - "dsw": "text/plain", - "dtd": "text/xml", - "dtsConfig": "text/xml", - "dv": "video/x-dv", - "dvi": "application/x-dvi", - "dwf": "drawing/x-dwf", - "dwp": "application/octet-stream", - "dxr": "application/x-director", - "eml": "message/rfc822", - "emz": "application/octet-stream", - "eot": "application/octet-stream", - "eps": "application/postscript", - "etl": "application/etl", - "etx": "text/x-setext", - "evy": "application/envoy", - "exe": "application/octet-stream", - "exe.config": "text/xml", - "fdf": "application/vnd.fdf", - "fif": "application/fractals", - "filters": "Application/xml", - "fla": "application/octet-stream", - "flr": "x-world/x-vrml", - "flv": "video/x-flv", - "fsscript": "application/fsharp-script", - "fsx": "application/fsharp-script", - "generictest": "application/xml", - "gif": "image/gif", - "group": "text/x-ms-group", - "gsm": "audio/x-gsm", - "gtar": "application/x-gtar", - "gz": "application/x-gzip", - "h": "text/plain", - "hdf": "application/x-hdf", - "hdml": "text/x-hdml", - "hhc": "application/x-oleobject", - "hhk": "application/octet-stream", - "hhp": "application/octet-stream", - "hlp": "application/winhlp", - "hpp": "text/plain", - "hqx": "application/mac-binhex40", - "hta": "application/hta", - "htc": "text/x-component", - "htm": "text/html", - "html": "text/html", - "htt": "text/webviewhtml", - "hxa": "application/xml", - "hxc": "application/xml", - "hxd": "application/octet-stream", - "hxe": "application/xml", - "hxf": "application/xml", - "hxh": "application/octet-stream", - "hxi": "application/octet-stream", - "hxk": "application/xml", - "hxq": "application/octet-stream", - "hxr": "application/octet-stream", - "hxs": "application/octet-stream", - "hxt": "text/html", - "hxv": "application/xml", - "hxw": "application/octet-stream", - "hxx": "text/plain", - "i": "text/plain", - "ico": "image/x-icon", - "ics": "application/octet-stream", - "idl": "text/plain", - "ief": "image/ief", - "iii": "application/x-iphone", - "inc": "text/plain", - "inf": "application/octet-stream", - "inl": "text/plain", - "ins": "application/x-internet-signup", - "ipa": "application/x-itunes-ipa", - "ipg": "application/x-itunes-ipg", - "ipproj": "text/plain", - "ipsw": "application/x-itunes-ipsw", - "iqy": "text/x-ms-iqy", - "isp": "application/x-internet-signup", - "ite": "application/x-itunes-ite", - "itlp": "application/x-itunes-itlp", - "itms": "application/x-itunes-itms", - "itpc": "application/x-itunes-itpc", - "IVF": "video/x-ivf", - "jar": "application/java-archive", - "java": "application/octet-stream", - "jck": "application/liquidmotion", - "jcz": "application/liquidmotion", - "jfif": "image/pjpeg", - "jnlp": "application/x-java-jnlp-file", - "jpb": "application/octet-stream", - "jpe": "image/jpeg", - "jpeg": "image/jpeg", - "jpg": "image/jpeg", - "js": "application/x-javascript", - "json": "application/json", - "jsx": "text/jscript", - "jsxbin": "text/plain", - "latex": "application/x-latex", - "library-ms": "application/windows-library+xml", - "lit": "application/x-ms-reader", - "loadtest": "application/xml", - "lpk": "application/octet-stream", - "lsf": "video/x-la-asf", - "lst": "text/plain", - "lsx": "video/x-la-asf", - "lzh": "application/octet-stream", - "m13": "application/x-msmediaview", - "m14": "application/x-msmediaview", - "m1v": "video/mpeg", - "m2t": "video/vnd.dlna.mpeg-tts", - "m2ts": "video/vnd.dlna.mpeg-tts", - "m2v": "video/mpeg", - "m3u": "audio/x-mpegurl", - "m3u8": "audio/x-mpegurl", - "m4a": "audio/m4a", - "m4b": "audio/m4b", - "m4p": "audio/m4p", - "m4r": "audio/x-m4r", - "m4v": "video/x-m4v", - "mac": "image/x-macpaint", - "mak": "text/plain", - "man": "application/x-troff-man", - "manifest": "application/x-ms-manifest", - "map": "text/plain", - "master": "application/xml", - "mda": "application/msaccess", - "mdb": "application/x-msaccess", - "mde": "application/msaccess", - "mdp": "application/octet-stream", - "me": "application/x-troff-me", - "mfp": "application/x-shockwave-flash", - "mht": "message/rfc822", - "mhtml": "message/rfc822", - "mid": "audio/mid", - "midi": "audio/mid", - "mix": "application/octet-stream", - "mk": "text/plain", - "mmf": "application/x-smaf", - "mno": "text/xml", - "mny": "application/x-msmoney", - "mod": "video/mpeg", - "mov": "video/quicktime", - "movie": "video/x-sgi-movie", - "mp2": "video/mpeg", - "mp2v": "video/mpeg", - "mp3": "audio/mpeg", - "mp4": "video/mp4", - "mp4v": "video/mp4", - "mpa": "video/mpeg", - "mpe": "video/mpeg", - "mpeg": "video/mpeg", - "mpf": "application/vnd.ms-mediapackage", - "mpg": "video/mpeg", - "mpp": "application/vnd.ms-project", - "mpv2": "video/mpeg", - "mqv": "video/quicktime", - "ms": "application/x-troff-ms", - "msi": "application/octet-stream", - "mso": "application/octet-stream", - "mts": "video/vnd.dlna.mpeg-tts", - "mtx": "application/xml", - "mvb": "application/x-msmediaview", - "mvc": "application/x-miva-compiled", - "mxp": "application/x-mmxp", - "nc": "application/x-netcdf", - "nsc": "video/x-ms-asf", - "nws": "message/rfc822", - "ocx": "application/octet-stream", - "oda": "application/oda", - "odc": "text/x-ms-odc", - "odh": "text/plain", - "odl": "text/plain", - "odp": "application/vnd.oasis.opendocument.presentation", - "ods": "application/oleobject", - "odt": "application/vnd.oasis.opendocument.text", - "one": "application/onenote", - "onea": "application/onenote", - "onepkg": "application/onenote", - "onetmp": "application/onenote", - "onetoc": "application/onenote", - "onetoc2": "application/onenote", - "orderedtest": "application/xml", - "osdx": "application/opensearchdescription+xml", - "p10": "application/pkcs10", - "p12": "application/x-pkcs12", - "p7b": "application/x-pkcs7-certificates", - "p7c": "application/pkcs7-mime", - "p7m": "application/pkcs7-mime", - "p7r": "application/x-pkcs7-certreqresp", - "p7s": "application/pkcs7-signature", - "pbm": "image/x-portable-bitmap", - "pcast": "application/x-podcast", - "pct": "image/pict", - "pcx": "application/octet-stream", - "pcz": "application/octet-stream", - "pdf": "application/pdf", - "pfb": "application/octet-stream", - "pfm": "application/octet-stream", - "pfx": "application/x-pkcs12", - "pgm": "image/x-portable-graymap", - "pic": "image/pict", - "pict": "image/pict", - "pkgdef": "text/plain", - "pkgundef": "text/plain", - "pko": "application/vnd.ms-pki.pko", - "pls": "audio/scpls", - "pma": "application/x-perfmon", - "pmc": "application/x-perfmon", - "pml": "application/x-perfmon", - "pmr": "application/x-perfmon", - "pmw": "application/x-perfmon", - "png": "image/png", - "pnm": "image/x-portable-anymap", - "pnt": "image/x-macpaint", - "pntg": "image/x-macpaint", - "pnz": "image/png", - "pot": "application/vnd.ms-powerpoint", - "potm": "application/vnd.ms-powerpoint.template.macroEnabled.12", - "potx": "application/vnd.openxmlformats-officedocument.presentationml.template", - "ppa": "application/vnd.ms-powerpoint", - "ppam": "application/vnd.ms-powerpoint.addin.macroEnabled.12", - "ppm": "image/x-portable-pixmap", - "pps": "application/vnd.ms-powerpoint", - "ppsm": "application/vnd.ms-powerpoint.slideshow.macroEnabled.12", - "ppsx": "application/vnd.openxmlformats-officedocument.presentationml.slideshow", - "ppt": "application/vnd.ms-powerpoint", - "pptm": "application/vnd.ms-powerpoint.presentation.macroEnabled.12", - "pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation", - "prf": "application/pics-rules", - "prm": "application/octet-stream", - "prx": "application/octet-stream", - "ps": "application/postscript", - "psc1": "application/PowerShell", - "psd": "application/octet-stream", - "psess": "application/xml", - "psm": "application/octet-stream", - "psp": "application/octet-stream", - "pub": "application/x-mspublisher", - "pwz": "application/vnd.ms-powerpoint", - "qht": "text/x-html-insertion", - "qhtm": "text/x-html-insertion", - "qt": "video/quicktime", - "qti": "image/x-quicktime", - "qtif": "image/x-quicktime", - "qtl": "application/x-quicktimeplayer", - "qxd": "application/octet-stream", - "ra": "audio/x-pn-realaudio", - "ram": "audio/x-pn-realaudio", - "rar": "application/octet-stream", - "ras": "image/x-cmu-raster", - "rat": "application/rat-file", - "rc": "text/plain", - "rc2": "text/plain", - "rct": "text/plain", - "rdlc": "application/xml", - "resx": "application/xml", - "rf": "image/vnd.rn-realflash", - "rgb": "image/x-rgb", - "rgs": "text/plain", - "rm": "application/vnd.rn-realmedia", - "rmi": "audio/mid", - "rmp": "application/vnd.rn-rn_music_package", - "roff": "application/x-troff", - "rpm": "audio/x-pn-realaudio-plugin", - "rqy": "text/x-ms-rqy", - "rtf": "application/rtf", - "rtx": "text/richtext", - "ruleset": "application/xml", - "s": "text/plain", - "safariextz": "application/x-safari-safariextz", - "scd": "application/x-msschedule", - "sct": "text/scriptlet", - "sd2": "audio/x-sd2", - "sdp": "application/sdp", - "sea": "application/octet-stream", - "searchConnector-ms": "application/windows-search-connector+xml", - "setpay": "application/set-payment-initiation", - "setreg": "application/set-registration-initiation", - "settings": "application/xml", - "sgimb": "application/x-sgimb", - "sgml": "text/sgml", - "sh": "application/x-sh", - "shar": "application/x-shar", - "shtml": "text/html", - "sit": "application/x-stuffit", - "sitemap": "application/xml", - "skin": "application/xml", - "sldm": "application/vnd.ms-powerpoint.slide.macroEnabled.12", - "sldx": "application/vnd.openxmlformats-officedocument.presentationml.slide", - "slk": "application/vnd.ms-excel", - "sln": "text/plain", - "slupkg-ms": "application/x-ms-license", - "smd": "audio/x-smd", - "smi": "application/octet-stream", - "smx": "audio/x-smd", - "smz": "audio/x-smd", - "snd": "audio/basic", - "snippet": "application/xml", - "snp": "application/octet-stream", - "sol": "text/plain", - "sor": "text/plain", - "spc": "application/x-pkcs7-certificates", - "spl": "application/futuresplash", - "src": "application/x-wais-source", - "srf": "text/plain", - "SSISDeploymentManifest": "text/xml", - "ssm": "application/streamingmedia", - "sst": "application/vnd.ms-pki.certstore", - "stl": "application/vnd.ms-pki.stl", - "sv4cpio": "application/x-sv4cpio", - "sv4crc": "application/x-sv4crc", - "svc": "application/xml", - "swf": "application/x-shockwave-flash", - "t": "application/x-troff", - "tar": "application/x-tar", - "tcl": "application/x-tcl", - "testrunconfig": "application/xml", - "testsettings": "application/xml", - "tex": "application/x-tex", - "texi": "application/x-texinfo", - "texinfo": "application/x-texinfo", - "tgz": "application/x-compressed", - "thmx": "application/vnd.ms-officetheme", - "thn": "application/octet-stream", - "tif": "image/tiff", - "tiff": "image/tiff", - "tlh": "text/plain", - "tli": "text/plain", - "toc": "application/octet-stream", - "tr": "application/x-troff", - "trm": "application/x-msterminal", - "trx": "application/xml", - "ts": "video/vnd.dlna.mpeg-tts", - "tsv": "text/tab-separated-values", - "ttf": "application/octet-stream", - "tts": "video/vnd.dlna.mpeg-tts", - "txt": "text/plain", - "u32": "application/octet-stream", - "uls": "text/iuls", - "user": "text/plain", - "ustar": "application/x-ustar", - "vb": "text/plain", - "vbdproj": "text/plain", - "vbk": "video/mpeg", - "vbproj": "text/plain", - "vbs": "text/vbscript", - "vcf": "text/x-vcard", - "vcproj": "Application/xml", - "vcs": "text/plain", - "vcxproj": "Application/xml", - "vddproj": "text/plain", - "vdp": "text/plain", - "vdproj": "text/plain", - "vdx": "application/vnd.ms-visio.viewer", - "vml": "text/xml", - "vscontent": "application/xml", - "vsct": "text/xml", - "vsd": "application/vnd.visio", - "vsi": "application/ms-vsi", - "vsix": "application/vsix", - "vsixlangpack": "text/xml", - "vsixmanifest": "text/xml", - "vsmdi": "application/xml", - "vspscc": "text/plain", - "vss": "application/vnd.visio", - "vsscc": "text/plain", - "vssettings": "text/xml", - "vssscc": "text/plain", - "vst": "application/vnd.visio", - "vstemplate": "text/xml", - "vsto": "application/x-ms-vsto", - "vsw": "application/vnd.visio", - "vsx": "application/vnd.visio", - "vtx": "application/vnd.visio", - "wav": "audio/wav", - "wave": "audio/wav", - "wax": "audio/x-ms-wax", - "wbk": "application/msword", - "wbmp": "image/vnd.wap.wbmp", - "wcm": "application/vnd.ms-works", - "wdb": "application/vnd.ms-works", - "wdp": "image/vnd.ms-photo", - "webarchive": "application/x-safari-webarchive", - "webtest": "application/xml", - "wiq": "application/xml", - "wiz": "application/msword", - "wks": "application/vnd.ms-works", - "WLMP": "application/wlmoviemaker", - "wlpginstall": "application/x-wlpg-detect", - "wlpginstall3": "application/x-wlpg3-detect", - "wm": "video/x-ms-wm", - "wma": "audio/x-ms-wma", - "wmd": "application/x-ms-wmd", - "wmf": "application/x-msmetafile", - "wml": "text/vnd.wap.wml", - "wmlc": "application/vnd.wap.wmlc", - "wmls": "text/vnd.wap.wmlscript", - "wmlsc": "application/vnd.wap.wmlscriptc", - "wmp": "video/x-ms-wmp", - "wmv": "video/x-ms-wmv", - "wmx": "video/x-ms-wmx", - "wmz": "application/x-ms-wmz", - "wpl": "application/vnd.ms-wpl", - "wps": "application/vnd.ms-works", - "wri": "application/x-mswrite", - "wrl": "x-world/x-vrml", - "wrz": "x-world/x-vrml", - "wsc": "text/scriptlet", - "wsdl": "text/xml", - "wvx": "video/x-ms-wvx", - "x": "application/directx", - "xaf": "x-world/x-vrml", - "xaml": "application/xaml+xml", - "xap": "application/x-silverlight-app", - "xbap": "application/x-ms-xbap", - "xbm": "image/x-xbitmap", - "xdr": "text/plain", - "xht": "application/xhtml+xml", - "xhtml": "application/xhtml+xml", - "xla": "application/vnd.ms-excel", - "xlam": "application/vnd.ms-excel.addin.macroEnabled.12", - "xlc": "application/vnd.ms-excel", - "xld": "application/vnd.ms-excel", - "xlk": "application/vnd.ms-excel", - "xll": "application/vnd.ms-excel", - "xlm": "application/vnd.ms-excel", - "xls": "application/vnd.ms-excel", - "xlsb": "application/vnd.ms-excel.sheet.binary.macroEnabled.12", - "xlsm": "application/vnd.ms-excel.sheet.macroEnabled.12", - "xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", - "xlt": "application/vnd.ms-excel", - "xltm": "application/vnd.ms-excel.template.macroEnabled.12", - "xltx": "application/vnd.openxmlformats-officedocument.spreadsheetml.template", - "xlw": "application/vnd.ms-excel", - "xml": "text/xml", - "xmta": "application/xml", - "xof": "x-world/x-vrml", - "XOML": "text/plain", - "xpm": "image/x-xpixmap", - "xps": "application/vnd.ms-xpsdocument", - "xrm-ms": "text/xml", - "xsc": "application/xml", - "xsd": "text/xml", - "xsf": "text/xml", - "xsl": "text/xml", - "xslt": "text/xml", - "xsn": "application/octet-stream", - "xss": "application/xml", - "xtp": "application/octet-stream", - "xwd": "image/x-xwindowdump", - "z": "application/x-compress", - "zip": "application/x-zip-compressed" -} From 23a7f19d5b05ad5a77cc09a468349a615dcd71ce Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Tue, 7 Jul 2015 16:30:30 -0700 Subject: [PATCH 055/453] module passes typechk --- multipart/src/client.rs | 326 ---------------------------------- multipart/src/client/hyper.rs | 45 +++++ multipart/src/client/mod.rs | 220 +++++++++++++++++++++++ multipart/src/client/sized.rs | 66 +++++++ multipart/src/lib.rs | 14 +- 5 files changed, 340 insertions(+), 331 deletions(-) delete mode 100644 multipart/src/client.rs create mode 100644 multipart/src/client/hyper.rs create mode 100644 multipart/src/client/mod.rs create mode 100644 multipart/src/client/sized.rs diff --git a/multipart/src/client.rs b/multipart/src/client.rs deleted file mode 100644 index 426a6fe55..000000000 --- a/multipart/src/client.rs +++ /dev/null @@ -1,326 +0,0 @@ -//! The client side implementation of `multipart/form-data` requests. -//! -//! Use this when sending POST requests with files to a server. -//! -//! `ChunkedMultipart` sends chunked requests (recommended), -//! while `SizedMultipart` sends sized requests. -//! -//! Both implement the `MultipartRequest` trait for their core API. -//! -//! Sized requests are more human-readable and use less bandwidth -//! (as chunking adds [significant visual noise and overhead][chunked-example]), -//! but they must be able to load their entirety, including the contents of all files -//! and streams, into memory so the request body can be measured and its size set -//! in the `Content-Length` header. -//! -//! You should really only use sized requests if you intend to inspect the data manually on the -//! server side, as it will produce a more human-readable request body. Also, of course, if the -//! server doesn't support chunked requests or otherwise rejects them. -//! -//! [chunked-example]: http://en.wikipedia.org/wiki/Chunked_transfer_encoding#Example -use mime::{Mime, TopLevel, SubLevel, Attr, Value}; - -use std::borrow::{Borrow, BorrowMut}; -use std::cell::Cell; - -use std::fs::File; -use std::io; -use std::io::prelude::*; - -use std::marker::PhantomData; - -pub type Multipart = ChunkedMultipart; - -/// The core API for all multipart requests. -/// -/// As part of the API contract, all errors in writing to the HTTP stream are stashed until the -/// implementor's concrete `.send()` method is called, or if the last error is inspected with -/// `.last_err()`. -/// -/// However, no reading or writing is performed after an error occurs, to avoid wasting CPU cycles -/// on a request that will not complete, with the exception of API calls after the last error is -/// cleared with `.take_err()`. -pub trait MultipartRequest { - #[doc(hidden)] - type Stream: Write; - - type Error: From; - - #[doc(hidden)] - fn stream_mut(&mut self) -> &mut Self::Stream; - #[doc(hidden)] - fn write_boundary(&mut self) -> io::Result<()>; - - /// Get a reference to the last error returned from writing to the HTTP stream, if any. - fn last_err(&self) -> Option<&Self::Error>; - - /// Remove and return the last error to occur, allowing subsequent API calls to proceed - /// normally. - fn take_err(&mut self) -> Option; - - /// Write a text field to this multipart request. - /// `name` and `val` can be either owned `String` or `&str`. - fn write_text, V: Borrow>(mut self, name: N, val: V) -> Self { - if self.last_err.is_none() { - self.last_err = chain_result! { - self.write_field_headers(name, None, None), - self.write_line(val.borrow()), - self.write_boundary() - }.err().map(Self::Error::from) - } - - self - } - - /// Write a file to the multipart request, guessing its `Content-Type` - /// from its extension and supplying its filename. - /// - /// See `write_stream()` for more info. - fn write_file, F: BorrowMut>(&mut self, name: N, file: F) -> Self { - if self.last_err.is_none() { - self.last_err = chain_result! { - { // New borrow scope so we can reborrow `file` after - let file_path = file.borrow().path(); - let content_type = ::mime_guess::guess_mime_type(file_path); - self.write_field_headers(name.borrow(), file_path.filename_str(), content_type) - }, - io::copy(file.borrow_mut(), self.stream_mut()), - self.write_boundary() - }.err().map(Self::Error::from); - } - - self - } - - /// Write a byte stream to the multipart request as a file field, supplying `filename` if given, - /// and `content_type` if given or `application/octet-stream` if not. - /// - /// ##Warning - /// The given `Read` **must** be able to read to EOF (end of file/no more data), meaning - /// `Read::read()` returns `Ok(0)`. - /// If it never returns EOF it will be read to infinity and the request will never be completed. - /// - /// In the case of `SizedMultipart` this also can cause out-of-control memory usage as the - /// multipart data has to be written to an in-memory buffer so it can be measured. - /// - /// Use `Read::take` if you wish to send data from a `Read` that will never end otherwise. - fn write_stream, F: Borrow, Rt: Read, R: Borrow>( - mut self, name: N, read: R, filename: Option, content_type: Option - ) -> Self { - if self.last_err.is_none() { - let content_type = content_type.map_or_else(::mime_guess::octet_stream); - - self.last_err = chain_result! { - self.write_field_headers(name.borrow(), filename, content_type), - io::copy(read.borrow_mut(), self.stream_mut()), - self.write_boundary() - }.err().map(Self::Error::from); - } - - self - } - - #[doc(hidden)] - fn write_field_headers(&mut self, name: &str, filename: Option<&str>, content_type: Option) - -> io::Result<()> { - chain_result! { - write!(self.stream_mut(), "Content-Disposition: form-data; name=\"{}\"", name), - filename.map(|filename| write!(self.stream_mut(), "; filename=\"{}\"", filename)) - .unwrap_or(Ok(())), - content_type.map(|content_type| write!(self.stream_mut(), "\r\nContent-Type: {}", content_type)) - .unwrap_or(Ok(())), - self.write_line("\r\n") - } - } - - #[doc(hidden)] - fn write_line(&mut self, line: &str) -> io::Result<()> { - write!(self.stream_mut(), "{}\r\n", line) - } -} - -/// The entry point of the client-side multipart API. -/// -/// Add text fields with `.add_text()` and files with `.add_file()`, -/// then obtain a `hyper::client::Request` object and pass it to `.send()`. -pub struct ChunkedMultipart { - request: PhantomData, - stream: R::Stream, - boundary: String, - last_err: Option, -} - -impl ChunkedMultipart { - /// Create a new `ChunkedMultipart` to wrap a request. - /// - /// ##May Panic - /// If `req` fails sanity checks in `HttpRequest::apply_headers()`. - pub fn from_request(mut req: R) -> Result, R::Error> { - let boundary = ::gen_boundary(); - req.apply_headers(&boundary, None); - - let stream = try!(req.open_stream()); - - Multipart { - request: PhantomData, - stream: stream, - boundary: boundary, - last_err: None - } - } - - /// Finalize the request and return the response from the server. - pub fn send(self) -> <::Stream as HttpStream>::Result where R: HttpRequest { - self.last_err.and_then(|_| self.stream.finalize()) - } -} - -impl MultipartRequest for ChunkedMultipart { - type Stream = R::Stream; - type Error = R::Error; - - fn stream_mut(&mut self) -> &mut R::Stream { - &mut self.stream - } - - fn write_boundary(&mut self) -> io::Result<()> { - write!(self.stream, "{}\r\n", self.boundary) - } - - fn last_err(&self) -> Option<&Self::Error> { - self.last_err.as_ref() - } - - fn take_err(&mut self) -> Option { - self.last_err.take() - } -} - -/// A struct for sending a sized multipart request. The API varies subtly from `Multipart`. -/// -/// The request data will be written to a `Vec` so its size can be measured and the -/// `Content-Length` header set when the request is sent. -pub struct SizedMultipart { - data: Vec, - boundary: String, - last_err: Option, -} - -impl SizedMultipart { - pub fn new() -> SizedMultipart { - SizedMultipart { - data: Vec::new(), - boundary: ::gen_boundary(), - last_err: None, - } - } - - pub fn send(self, mut req: R) -> <::Stream as HttpStream>::Result - where R: HttpRequest { - let boundary = ::gen_boundary(); - req.apply_headers(&boundary, Some(self.data.len())); - let req = try!(req.open_stream()); - try!(io::copy(&self.data, &mut req)); - req.finish() - } -} - -impl MultipartRequest for SizedMultipart { - type Stream = Vec; - type Error = io::Error; - - #[doc(hidden)] - fn stream_mut(&mut self) -> &mut Vec { - &mut self.data - } - - #[doc(hidden)] - fn write_boundary(&mut self) -> io::Result<()> { - write!(self.stream, "{}\r\n", self.boundary) - } - - fn last_err(&self) -> Option<&Self::Error> { - self.last_err.as_ref() - } - - fn take_err(&mut self) -> Option { - self.last_err.take() - } -} - -fn multipart_mime(bound: &str) -> Mime { - Mime( - TopLevel::Multipart, SubLevel::Ext("form-data".into_string()), - vec![(Attr::Ext("boundary".into_string()), Value::Ext(bound.into_string()))] - ) -} - -pub trait HttpRequest { - type Stream: HttpStream; - type Response: Read; - type Error: From; - - /// Set the `ContentType` header to `multipart/form-data` and supply the `boundary` value. - /// - /// If `content_len` is given, set the `ContentLength` header to its value. - fn apply_headers(&mut self, boundary: &str, content_len: Option); - /// Open the request stream and invoke the given closure, where the request body will be - /// written. After the closure returns, finalize the request and return its result. - fn open_stream(self) -> Result; -} - -pub trait HttpStream: Write { - type Request: HttpRequest; - type Result = Result; - - /// Finalize and close the stream, returning the HTTP response object. - fn finish(self) -> Self::Result; -} - -#[cfg(feature = "hyper")] -mod hyper_impl { - use hyper::client::request::Request; - use hyper::client::response::Response; - use hyper::error::Error as HyperError; - use hyper::net::{Fresh, Streaming}; - - use std::io; - - impl super::HttpRequest for Request { - type Stream = Request; - type Response = Response; - type Error = HyperError; - - /// #Panics - /// If the `Request` method is not `Method::Post`. - fn apply_headers(&mut self, boundary: &str, content_len: Option) { - use hyper::header::{ContentType, ContentLength}; - use hyper::method::Method; - - assert!(self.method() == Method::Post, "Multipart request must use POST method!"); - - let headers = self.headers_mut(); - - headers.set(ContentType(super::multipart_mime(boundary))); - - if let Some(size) = content_len { - headers.set(ContentLength(size)); - } - - debug!("Hyper headers: {}", self.headers()); - } - - fn send(self, send_fn: F) -> Self::RequestResult - where F: FnOnce(&mut Request) -> io::Result<()> - { - let mut req = try!(self.start()); - try!(send_fn(&mut req)); - req.send() - } - } - - impl super::HttpStream for Request { - - } -} - diff --git a/multipart/src/client/hyper.rs b/multipart/src/client/hyper.rs new file mode 100644 index 000000000..dd7323f3e --- /dev/null +++ b/multipart/src/client/hyper.rs @@ -0,0 +1,45 @@ +use hyper::client::request::Request; +use hyper::client::response::Response; +use hyper::error::Error as HyperError; +use hyper::header::{ContentType, ContentLength}; +use hyper::method::Method; +use hyper::net::{Fresh, Streaming}; + +use super::{HttpRequest, HttpStream}; + +use std::io; +use std::io::prelude::*; + +impl super::HttpRequest for Request { + type Stream = Request; + type Response = Response; + type Error = HyperError; + + /// #Panics + /// If the `Request` method is not `Method::Post`. + fn apply_headers(&mut self, boundary: &str, content_len: Option) { + + assert!(self.method() == Method::Post, "Multipart request must use POST method!"); + + let headers = self.headers_mut(); + + headers.set(ContentType(super::multipart_mime(boundary))); + + if let Some(size) = content_len { + headers.set(ContentLength(size)); + } + + debug!("Hyper headers: {}", self.headers()); + } + + fn send(self, send_fn: F) -> Self::RequestResult + where F: FnOnce(&mut Request) -> io::Result<()> { + let mut req = try!(self.start()); + try!(send_fn(&mut req)); + req.send() + } +} + +impl super::HttpStream for Request { + +} diff --git a/multipart/src/client/mod.rs b/multipart/src/client/mod.rs new file mode 100644 index 000000000..3c1a64c7c --- /dev/null +++ b/multipart/src/client/mod.rs @@ -0,0 +1,220 @@ +//! The client side implementation of `multipart/form-data` requests. +//! +//! Use this when sending POST requests with files to a server. +//! +//! `ChunkedMultipart` sends chunked requests (recommended), +//! while `SizedMultipart` sends sized requests. +//! +//! Both implement the `MultipartRequest` trait for their core API. +//! +//! Sized requests are more human-readable and use less bandwidth +//! (as chunking adds [significant visual noise and overhead][chunked-example]), +//! but they must be able to load their entirety, including the contents of all files +//! and streams, into memory so the request body can be measured and its size set +//! in the `Content-Length` header. +//! +//! You should really only use sized requests if you intend to inspect the data manually on the +//! server side, as it will produce a more human-readable request body. Also, of course, if the +//! server doesn't support chunked requests or otherwise rejects them. +//! +//! [chunked-example]: http://en.wikipedia.org/wiki/Chunked_transfer_encoding#Example +use mime::{Mime, TopLevel, SubLevel, Attr, Value}; + +use std::borrow::{Borrow, BorrowMut}; +use std::cell::Cell; +use std::convert::AsRef; + +use std::fs::File; +use std::io; +use std::io::prelude::*; + +use std::path::Path; + +#[cfg(feature = "hyper")] +mod hyper; + +mod sized; + +pub use self::sized::SizedRequest; + + +/// The entry point of the client-side multipart API. +/// +/// Though they perform I/O, the `.write_*()` methods do not return `io::Result<_>` in order to +/// facilitate method chaining. Upon the first error, all subsequent API calls will be no-ops until +/// `.send()` is called, at which point the error will be reported. +/// +/// If you don't want to consume data handles (`File`, etc.), `write_file()` and `write_stream()` +/// also accept `&mut` versions. +pub struct Multipart { + stream: S, + boundary: String, + last_err: Option, +} + +impl Multipart { + /// Create a new `Multipart` to wrap a request. + /// + /// ## Returns Error + /// If `req.open_stream()` returns an error. + pub fn from_request(mut req: R) -> Result, R::Error> { + let boundary = ::gen_boundary(); + req.apply_headers(&boundary, None); + + let stream = try!(req.open_stream()); + + Ok(Multipart { + stream: stream, + boundary: boundary, + last_err: None + }) + } +} + +impl Multipart { + /// Get a reference to the last error returned from writing to the HTTP stream, if any. + pub fn last_err(&self) -> Option<&S::Error> { + self.last_err.as_ref() + } + + /// Remove and return the last error to occur, allowing subsequent API calls to proceed + /// normally. + pub fn take_err(&mut self) -> Option { + self.last_err.take() + } + + /// Write a text field to this multipart request. + /// `name` and `val` can be either owned `String` or `&str`. + pub fn write_text, V: Borrow>(mut self, name: N, val: V) -> Self { + if self.last_err.is_none() { + self.last_err = chain_result! { + self.write_field_headers(name.borrow(), None, None), + self.write_line(val.borrow()), + self.write_boundary() + }.err().map(|err| err.into()) + } + + self + } + + /// Open a file pointed to by `path` and write its contents to the multipart request, + /// supplying its filename and guessing its `Content-Type` from its extension. + /// + /// If you want to set these values manually, or use another type that implements `Read`, + /// use `.write_stream()`. + // Remove `File::path()`, GG Rust-lang + pub fn write_file, P: AsRef>(mut self, name: N, path: P) -> Self { + if self.last_err.is_none() { + let path = path.as_ref(); + + self.last_err = chain_result! { + { // New borrow scope so we can reborrow `file` after + let content_type = ::mime_guess::guess_mime_type(path); + let filename = path.file_name().and_then(|filename| filename.to_str()); + self.write_field_headers(name.borrow(), filename, Some(content_type)) + }, + File::open(path).and_then(|ref mut file| io::copy(file, &mut self.stream)), + self.write_boundary() + }.err().map(|err| err.into()); + } + + self + } + + /// Write a byte stream to the multipart request as a file field, supplying `filename` if given, + /// and `content_type` if given or `application/octet-stream` if not. + /// + /// ##Warning + /// The given `Read` **must** be able to read to EOF (end of file/no more data), meaning + /// `Read::read()` returns `Ok(0)`. If it never returns EOF it will be read to infinity + /// and the request will never be completed. + /// + /// When using `sized::SizedRequest` this also can cause out-of-control memory usage as the + /// multipart data has to be written to an in-memory buffer so its size can be calculated. + /// + /// Use `Read::take` if you wish to send data from a `Read` that will never end otherwise. + // RFC: How to format this declaration? + pub fn write_stream, St: Read, St_: BorrowMut>( + mut self, name: N, read: St_, filename: Option<&str>, content_type: Option + ) -> Self { + if self.last_err.is_none() { + let content_type = content_type.unwrap_or_else(::mime_guess::octet_stream); + + self.last_err = chain_result! { + self.write_field_headers(name.borrow(), filename, Some(content_type)), + io::copy(read.borrow_mut(), &mut self.stream), + self.write_boundary() + }.err().map(|err| err.into()); + } + + self + } + + fn write_field_headers(&mut self, name: &str, filename: Option<&str>, content_type: Option) + -> io::Result<()> { + chain_result! { + write!(self.stream, "Content-Disposition: form-data; name=\"{}\"", name), + filename.map(|filename| write!(self.stream, "; filename=\"{}\"", filename)) + .unwrap_or(Ok(())), + content_type.map(|content_type| write!(self.stream, "\r\nContent-Type: {}", content_type)) + .unwrap_or(Ok(())), + self.write_line("\r\n") + } + } + + fn write_line(&mut self, line: &str) -> io::Result<()> { + write!(self.stream, "{}\r\n", line) + } + + fn write_boundary(&mut self) -> io::Result<()> { + write!(self.stream, "{}\r\n", self.boundary) + } + + /// Finalize the request and return the response from the server. + // God bless this mess everyone! + pub fn send(self) -> Result { + match self.last_err { + None => self.stream.finish(), + Some(err) => Err(err), + } + } +} + +impl Multipart> +where ::Error: From { + /// Create a new `Multipart` using the `SizedRequest` wrapper around `req`. + pub fn from_request_sized(req: R) -> Result { + Multipart::>::from_request(SizedRequest::from_request(req)) + } +} + + +fn multipart_mime(bound: &str) -> Mime { + Mime( + TopLevel::Multipart, SubLevel::Ext("form-data".into()), + vec![(Attr::Ext("boundary".into()), Value::Ext(bound.into()))] + ) +} + +pub trait HttpRequest { + type Stream: HttpStream; + type Error: From + Into<::Error>; + + /// Set the `ContentType` header to `multipart/form-data` and supply the `boundary` value. + /// If `content_len` is given, set the `Content-Length` header to its value. + fn apply_headers(&mut self, boundary: &str, content_len: Option); + + /// Open the request stream and return it, after which point the request body will be + /// written. `HttpStream::finish()` will be called after the body has finished being written. + fn open_stream(self) -> Result; +} + +pub trait HttpStream: Write { + type Request: HttpRequest; + type Response: Read; + type Error: From + From<::Error>; + + /// Finalize and close the stream, returning the HTTP response object. + fn finish(self) -> Result; +} + diff --git a/multipart/src/client/sized.rs b/multipart/src/client/sized.rs new file mode 100644 index 000000000..225c39b43 --- /dev/null +++ b/multipart/src/client/sized.rs @@ -0,0 +1,66 @@ +//! Sized/buffered wrapper around `HttpRequest`. + +use client::{HttpRequest, HttpStream}; + +use std::io; +use std::io::prelude::*; + +/// A wrapper around `HttpRequest` that writes the multipart data to an in-memory buffer so its +/// size can be calculated and set in the request as the `Content-Length` header. +pub struct SizedRequest { + inner: R, + buffer: Vec, + boundary: String, +} + +impl SizedRequest { + #[doc(hidden)] + pub fn from_request(req: R) -> SizedRequest { + SizedRequest { + inner: req, + buffer: Vec::new(), + boundary: String::new(), + } + } +} + +impl Write for SizedRequest { + fn write(&mut self, data: &[u8]) -> io::Result { + self.buffer.write(data) + } + + fn flush(&mut self) -> io::Result<()> { Ok(()) } +} + +impl HttpRequest for SizedRequest +where ::Error: From { + type Stream = Self; + type Error = R::Error; + + /// `SizedRequest` ignores `_content_len` because it sets its own later. + fn apply_headers(&mut self, boundary: &str, _content_len: Option) { + self.boundary.clear(); + self.boundary.push_str(boundary); + } + + fn open_stream(mut self) -> Result { + self.buffer.clear(); + Ok(self) + } +} + +impl HttpStream for SizedRequest +where ::Error: From { + type Request = R; + type Response = <::Stream as HttpStream>::Response; + type Error = <::Stream as HttpStream>::Error; + + fn finish(mut self) -> Result { + let content_len = self.buffer.len(); + self.inner.apply_headers(&self.boundary, Some(content_len)); + + let mut req = try!(self.inner.open_stream()); + try!(io::copy(&mut &self.buffer[..], &mut req)); + req.finish().into() + } +} diff --git a/multipart/src/lib.rs b/multipart/src/lib.rs index 2af4dc256..23aa50396 100644 --- a/multipart/src/lib.rs +++ b/multipart/src/lib.rs @@ -10,6 +10,8 @@ extern crate hyper; use mime::Mime; +use rand::Rng; + use std::borrow::Cow; use std::fmt; @@ -33,12 +35,14 @@ macro_rules! chain_result { pub mod client; pub mod server; - - const BOUNDARY_LEN: usize = 16; -fn gen_boundary() -> String { - use rand::Rng; +fn random_alphanumeric(len: usize) -> String { + rand::thread_rng().gen_ascii_chars().take(len).collect() +} - "--".chars().chain(rand::thread_rng().gen_ascii_chars().take(BOUNDARY_LEN)).collect() +fn gen_boundary() -> String { + let mut boundary = "--".to_owned(); + boundary.extend(rand::thread_rng().gen_ascii_chars().take(BOUNDARY_LEN)); + boundary } From 9245f1df9a41858697ae646201f846ae0745e434 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Tue, 7 Jul 2015 22:50:59 -0700 Subject: [PATCH 056/453] What?! Everyting compiles now?! Wait, tests don't yet... --- multipart/src/client/mod.rs | 2 +- multipart/src/lib.rs | 10 --- multipart/src/server/boundary.rs | 27 ++++--- multipart/src/server/mod.rs | 129 +++++++++++++++---------------- 4 files changed, 82 insertions(+), 86 deletions(-) diff --git a/multipart/src/client/mod.rs b/multipart/src/client/mod.rs index 3c1a64c7c..2e724371e 100644 --- a/multipart/src/client/mod.rs +++ b/multipart/src/client/mod.rs @@ -135,7 +135,7 @@ impl Multipart { /// Use `Read::take` if you wish to send data from a `Read` that will never end otherwise. // RFC: How to format this declaration? pub fn write_stream, St: Read, St_: BorrowMut>( - mut self, name: N, read: St_, filename: Option<&str>, content_type: Option + mut self, name: N, mut read: St_, filename: Option<&str>, content_type: Option ) -> Self { if self.last_err.is_none() { let content_type = content_type.unwrap_or_else(::mime_guess::octet_stream); diff --git a/multipart/src/lib.rs b/multipart/src/lib.rs index 23aa50396..fc68eb0e6 100644 --- a/multipart/src/lib.rs +++ b/multipart/src/lib.rs @@ -8,18 +8,8 @@ extern crate rustc_serialize; #[cfg(feature = "hyper")] extern crate hyper; -use mime::Mime; - use rand::Rng; -use std::borrow::Cow; -use std::fmt; - -use std::fs::File; -use std::io; -use std::io::prelude::*; -use std::path::{Path, PathBuf}; - macro_rules! try_all { ($first_expr:expr, $($try_expr:expr),*) => ( try!($first_expr $(.and_then(|_| $try_expr))*); diff --git a/multipart/src/server/boundary.rs b/multipart/src/server/boundary.rs index be5c8506e..690b5d68d 100644 --- a/multipart/src/server/boundary.rs +++ b/multipart/src/server/boundary.rs @@ -7,6 +7,7 @@ use std::io::prelude::*; use std::ptr; /// A struct implementing `Read` that will yield bytes until it sees a given sequence. +#[derive(Debug)] pub struct BoundaryReader { reader: S, boundary: Vec, @@ -19,7 +20,8 @@ pub struct BoundaryReader { const BUF_SIZE: usize = 1024 * 64; // 64k buffer impl BoundaryReader where S: Read { - fn from_reader(reader: S, boundary: String) -> BoundaryReader { + #[doc(hidden)] + pub fn from_reader(reader: S, boundary: String) -> BoundaryReader { let mut buf = vec![0u8; BUF_SIZE]; BoundaryReader { @@ -40,7 +42,7 @@ impl BoundaryReader where S: Read { let lookahead = &self.buf[self.last_search_idx .. self.buf_len]; - let search_idx = lookahead.position_elem(&self.boundary[0]) + let search_idx = lookahead.iter().position(|&byte| byte == self.boundary[0]) .unwrap_or(lookahead.len() - 1); debug!("Search idx: {}", search_idx); @@ -84,7 +86,8 @@ impl BoundaryReader where S: Read { self.last_search_idx -= amt; } - fn consume_boundary(&mut self) -> io::Result<()> { + #[doc(hidden)] + pub fn consume_boundary(&mut self) -> io::Result<()> { while !self.boundary_read { try!(self.read_to_boundary()); } @@ -172,24 +175,30 @@ dashed-value-2\r let test_reader = BufReader::new(TEST_VAL.as_bytes()); let mut reader = BoundaryReader::from_reader(test_reader, BOUNDARY.to_owned()); + let ref mut buf = String::new(); + debug!("Read 1"); - let string = reader.read_to_string().unwrap(); - debug!("{}", string); - assert!(string.trim().is_empty()); + let _ = reader.read_to_string(buf).unwrap(); + debug!("{}", buf); + assert!(buf.trim().is_empty()); + + buf.clear(); debug!("Consume 1"); reader.consume_boundary().unwrap(); debug!("Read 2"); - assert_eq!(reader.read_to_string().unwrap().trim(), "dashed-value-1"); + let _ = reader.read_to_string(buf).unwrap(); + assert_eq!(buf.trim(), "dashed-value-1"); + buf.clear(); debug!("Consume 2"); reader.consume_boundary().unwrap(); debug!("Read 3"); - assert_eq!(reader.read_to_string().unwrap().trim(), "dashed-value-2"); + let _ = reader.read_to_string(buf).unwrap(); + assert_eq!(buf.trim(), "dashed-value-2"); debug!("Consume 3"); reader.consume_boundary().unwrap(); - } diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index 3df4a5390..f23081ea5 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -14,7 +14,7 @@ use std::ops::Deref; use std::fmt; -use std::fs::File; +use std::fs::{self, File}; use std::io; use std::io::prelude::*; @@ -53,10 +53,11 @@ impl Multipart where R: HttpRequest { pub fn from_request(req: R) -> Result, R> { if !req.is_multipart() { return Err(req); } - let boundary = match req.get_boundary() { - Some(boundary) => format!("{}\r\n", boundary), - None => return Err(req), - }; + if req.get_boundary().is_none() { + return Err(req); + } + + let boundary = req.get_boundary().unwrap().into(); debug!("Boundary: {}", boundary); @@ -76,7 +77,7 @@ impl Multipart where R: HttpRequest { /// ##Warning /// If the last returned entry had contents of type `MultipartField::File`, /// calling this again will discard any unread contents of that entry! - pub fn read_entry(&mut self) -> io::Result<(String, MultipartField)> { + pub fn read_entry(&mut self) -> io::Result<(String, MultipartField)> { try!(self.source.consume_boundary()); let (disp_type, field_name, filename) = try!(self.read_content_disposition()); @@ -91,7 +92,7 @@ impl Multipart where R: HttpRequest { let _ = try!(self.read_line()); // Consume empty line Ok((field_name, MultipartField::File( - MultipartFile::from_octet(filename, &mut self.source, &content_type) + MultipartFile::from_reader(filename, &mut self.source, &content_type) ) )) } else { @@ -109,7 +110,7 @@ impl Multipart where R: HttpRequest { /// since `Iterator::next()` can't use bound lifetimes. /// /// See https://www.reddit.com/r/rust/comments/2lkk\4\isize/concrete_lifetime_vs_bound_lifetime/ - pub fn foreach_entry(&mut self, mut foreach: F) where F: FnMut(String, MultipartField) { + pub fn foreach_entry(&mut self, mut foreach: F) where F: FnMut(String, MultipartField) { loop { match self.read_entry() { Ok((name, field)) => foreach(name, field), @@ -184,7 +185,7 @@ impl Multipart where R: HttpRequest { loop { match self.read_entry() { - Ok((name, MultipartField::Text(text))) => { entries.fields.insert(name, text.into_owned()); }, + Ok((name, MultipartField::Text(text))) => { entries.fields.insert(name, text.to_owned()); }, Ok((name, MultipartField::File(mut file))) => { let path = try!(file.save_in(&entries.dir)); entries.files.insert(name, path); @@ -200,11 +201,17 @@ impl Multipart where R: HttpRequest { } fn read_line(&mut self) -> io::Result<&str> { - self.source.read_line(&mut self.line_buf).map(|read| &self.line_buf[..read]) + match self.source.read_line(&mut self.line_buf) { + Ok(read) => Ok(&self.line_buf[..read]), + Err(err) => Err(err), + } } fn read_to_string(&mut self) -> io::Result<&str> { - self.source.read_to_string(&mut self.line_buf).map(|read| &self.line_buf[..read]) + match self.source.read_to_string(&mut self.line_buf) { + Ok(read) => Ok(&self.line_buf[..read]), + Err(err) => Err(err), + } } } @@ -253,6 +260,39 @@ pub trait HttpRequest: Read { fn get_boundary(&self) -> Option<&str>; } +/// A field in a `multipart/form-data` request. +/// +/// This enum does not include the names of the fields, as those are yielded separately +/// by `server::Multipart::read_entry()`. +#[derive(Debug)] +pub enum MultipartField<'a, R: 'a> { + /// A text field. + Text(&'a str), + /// A file field, including the content type and optional filename + /// along with a `Read` implementation for getting the contents. + File(MultipartFile<'a, R>), + // MultiFiles(Vec), /* TODO: Multiple files */ +} + +impl<'a, R> MultipartField<'a, R> { + /// Borrow this field as a text field, if possible. + pub fn as_text(&self) -> Option<&str> { + match *self { + MultipartField::Text(ref s) => Some(s), + _ => None, + } + } + + /// Borrow this field as a file field, if possible + /// Mutably borrows so the contents can be read. + pub fn as_file(&mut self) -> Option<&mut MultipartFile<'a, R>> { + match *self { + MultipartField::File(ref mut file) => Some(file), + _ => None, + } + } +} + /// A representation of a file in HTTP `multipart/form-data`. /// /// Note that the file is not yet saved to the system; @@ -260,35 +300,30 @@ pub trait HttpRequest: Read { /// to the beginning of the file's contents in the HTTP stream. /// You can read it to EOF, or use one of the `save_*()` methods here /// to save it to disk. -pub struct MultipartFile<'a, R> { +#[derive(Debug)] +pub struct MultipartFile<'a, R: 'a> { filename: Option, content_type: Mime, reader: &'a mut BoundaryReader, } -impl<'a> MultipartFile<'a> { - fn from_octet(filename: Option, reader: &'a mut Read, cont_type: &str) -> MultipartFile<'a> { +impl<'a, R: Read> MultipartFile<'a, R> { + fn from_reader( + filename: Option, reader: &'a mut BoundaryReader, cont_type: &str + ) -> MultipartFile<'a, R> { MultipartFile { filename: filename, + content_type: cont_type.parse::().ok().unwrap_or_else(::mime_guess::octet_stream), reader: reader, - content_type: cont_type.parse::().ok().unwrap_or_else(::mime_guess::octet_stream), } } - fn from_file(filename: Option, reader: &'a mut File, mime: Mime) -> MultipartFile<'a> { - MultipartFile { - filename: filename, - reader: reader, - content_type: mime, - } - } - /// Save this file to `path`, discarding the filename. /// /// If successful, the file can be found at `path`. pub fn save_as(&mut self, path: &Path) -> io::Result<()> { let mut file = try!(File::create(path)); - io::copy(self.reader, &mut file) + io::copy(self.reader, &mut file).and(Ok(())) } /// Save this file in the directory described by `dir`, @@ -299,9 +334,10 @@ impl<'a> MultipartFile<'a> { /// ###Panics /// If `dir` does not represent a directory. pub fn save_in(&mut self, dir: &Path) -> io::Result { - assert!(dir.is_dir(), "Given path is not a directory!"); + let meta = try!(fs::metadata(dir)); + assert!(meta.is_dir(), "Given path is not a directory!"); - let path = dir.join(self.dest_filename()); + let path = dir.join(::random_alphanumeric(8)); try!(self.save_as(&path)); @@ -330,51 +366,12 @@ impl<'a> MultipartFile<'a> { } } -impl<'a> Read for MultipartFile<'a> { +impl<'a, R: Read> Read for MultipartFile<'a, R> { fn read(&mut self, buf: &mut [u8]) -> io::Result{ self.reader.read(buf) } } -impl<'a> fmt::Debug for MultipartFile<'a> { - fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { - write!(fmt, "Filename: {:?} Content-Type: {:?}", self.filename, self.content_type) - } -} - - -/// A field in a `multipart/form-data` request. -/// -/// This enum does not include the names of the fields, as those are yielded separately -/// by `server::Multipart::read_entry()`. -#[derive(Debug)] -pub enum MultipartField<'a> { - /// A text field. - Text(&'a str), - /// A file field, including the content type and optional filename - /// along with a `Read` implementation for getting the contents. - File(MultipartFile<'a>), - // MultiFiles(Vec), /* TODO: Multiple files */ -} - -impl<'a> MultipartField<'a> { - /// Borrow this field as a text field, if possible. - pub fn as_text(&self) -> Option<&str> { - match *self { - MultipartField::Text(ref s) => Some(s), - _ => None, - } - } - - /// Borrow this field as a file field, if possible - /// Mutably borrows so the contents can be read. - pub fn as_file(&mut self) -> Option<&mut MultipartFile<'a>> { - match *self { - MultipartField::File(ref mut file) => Some(file), - _ => None, - } - } -} From f4b6faf29447a6e1ffea2284e58d808a84a8caf1 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Wed, 8 Jul 2015 02:13:02 -0700 Subject: [PATCH 057/453] Compiles but fails test --- multipart/src/client/mod.rs | 1 - multipart/src/server/boundary.rs | 138 ++++++++++------------------ multipart/src/server/mod.rs | 2 +- multipart/tests/multipart_server.rs | 52 ----------- 4 files changed, 51 insertions(+), 142 deletions(-) delete mode 100644 multipart/tests/multipart_server.rs diff --git a/multipart/src/client/mod.rs b/multipart/src/client/mod.rs index 2e724371e..2617a3d49 100644 --- a/multipart/src/client/mod.rs +++ b/multipart/src/client/mod.rs @@ -171,7 +171,6 @@ impl Multipart { } /// Finalize the request and return the response from the server. - // God bless this mess everyone! pub fn send(self) -> Result { match self.last_err { None => self.stream.finish(), diff --git a/multipart/src/server/boundary.rs b/multipart/src/server/boundary.rs index 690b5d68d..da2d1d423 100644 --- a/multipart/src/server/boundary.rs +++ b/multipart/src/server/boundary.rs @@ -2,146 +2,108 @@ use std::cmp; use std::borrow::Borrow; use std::io; +use std::io::BufReader; use std::io::prelude::*; use std::ptr; /// A struct implementing `Read` that will yield bytes until it sees a given sequence. #[derive(Debug)] -pub struct BoundaryReader { - reader: S, +pub struct BoundaryReader { + buffer: BufReader, boundary: Vec, - last_search_idx: usize, + search_idx: usize, boundary_read: bool, - buf: Vec, - buf_len: usize, } -const BUF_SIZE: usize = 1024 * 64; // 64k buffer -impl BoundaryReader where S: Read { +impl BoundaryReader where R: Read { #[doc(hidden)] - pub fn from_reader(reader: S, boundary: String) -> BoundaryReader { - let mut buf = vec![0u8; BUF_SIZE]; - + pub fn from_reader>>(reader: R, boundary: B) -> BoundaryReader { BoundaryReader { - reader: reader, - boundary: boundary.into_bytes(), - last_search_idx: 0, + buffer: BufReader::new(reader), + boundary: boundary.into(), + search_idx: 0, boundary_read: false, - buf: buf, - buf_len: 0, } } - fn read_to_boundary(&mut self) -> io::Result<()> { - if !self.boundary_read { - try!(self.true_fill_buf()); - - if self.buf_len == 0 { return Ok(()); } - - let lookahead = &self.buf[self.last_search_idx .. self.buf_len]; - - let search_idx = lookahead.iter().position(|&byte| byte == self.boundary[0]) - .unwrap_or(lookahead.len() - 1); - - debug!("Search idx: {}", search_idx); - - self.boundary_read = lookahead[search_idx..] - .starts_with(&self.boundary); + fn read_to_boundary(&mut self) -> io::Result<&[u8]> { + let buf = try!(self.buffer.fill_buf()); - self.last_search_idx += search_idx; + if !self.boundary_read { + let boundary_0 = self.boundary[0]; - if !self.boundary_read { - self.last_search_idx += 1; + // Style advice requested + for (search_idx, maybe_boundary) + in buf[self.search_idx..].windows(self.boundary.len()).enumerate() + .filter(|&(_, window)| window[0] == boundary_0) + { + self.boundary_read = self.boundary == maybe_boundary; + self.search_idx += search_idx; } } - Ok(()) - } - - /// Read bytes until the reader is full - fn true_fill_buf(&mut self) -> io::Result<()> { - let mut bytes_read = 0; - - loop { - bytes_read = try!(self.reader.read(&mut self.buf[self.buf_len..])); - if bytes_read == 0 { break; } - self.buf_len += bytes_read; - } - - Ok(()) - } - - fn _consume(&mut self, amt: usize) { - use std::ptr; - - assert!(amt <= self.buf_len); - - let (dest, src) = self.buf.split_at_mut(amt); - - copy_bytes(src, dest); - - self.buf_len -= amt; - self.last_search_idx -= amt; + Ok(&buf[..self.search_idx]) } #[doc(hidden)] pub fn consume_boundary(&mut self) -> io::Result<()> { while !self.boundary_read { - try!(self.read_to_boundary()); + let buf_len = try!(self.read_to_boundary()).len(); + self.consume(buf_len); } - let consume_amt = cmp::min(self.buf_len, self.last_search_idx + self.boundary.len()); + let consume_amt = { + let boundary_len = self.boundary.len(); + let buf = try!(self.read_to_boundary()); + buf.len() + boundary_len + }; - debug!("Consume amt: {} Buf len: {}", consume_amt, self.buf_len); - - self._consume(consume_amt); - self.last_search_idx = 0; + self.buffer.consume(consume_amt); + self.search_idx = 0; self.boundary_read = false; Ok(()) } #[allow(unused)] - fn set_boundary(&mut self, boundary: String) { - self.boundary = boundary.into_bytes(); + #[doc(hidden)] + pub fn set_boundary>>(&mut self, boundary: B) { + self.boundary = boundary.into(); } } -impl Borrow for BoundaryReader { +impl Borrow for BoundaryReader { fn borrow(&self) -> &R { - &self.reader + self.buffer.get_ref() } } impl Read for BoundaryReader where R: Read { - fn read(&mut self, buf: &mut [u8]) -> io::Result { - use std::cmp; - - try!(self.read_to_boundary()); + fn read(&mut self, out: &mut [u8]) -> io::Result { + let consume_len = { + let buf = try!(self.read_to_boundary()); + let trunc_len = cmp::min(buf.len(), out.len()); + copy_bytes(&buf[..trunc_len], out); + trunc_len + }; - let trunc_len = cmp::min(buf.len(), self.last_search_idx); - copy_bytes(&self.buf[..trunc_len], buf); + self.consume(consume_len); - self._consume(trunc_len); - - Ok(trunc_len) + Ok(consume_len) } } impl BufRead for BoundaryReader where R: Read { fn fill_buf(&mut self) -> io::Result<&[u8]> { - try!(self.read_to_boundary()); - - let buf = &self.buf[..self.last_search_idx]; - - Ok(buf) + self.read_to_boundary() } fn consume(&mut self, amt: usize) { - assert!(amt <= self.last_search_idx); - self._consume(amt); + let true_amt = cmp::min(amt, self.search_idx); + self.buffer.consume(true_amt); + self.search_idx -= true_amt; } } @@ -172,8 +134,8 @@ dashed-value-2\r --boundary\r "; - let test_reader = BufReader::new(TEST_VAL.as_bytes()); - let mut reader = BoundaryReader::from_reader(test_reader, BOUNDARY.to_owned()); + let src = &mut TEST_VAL.as_bytes(); + let mut reader = BoundaryReader::from_reader(src, BOUNDARY); let ref mut buf = String::new(); diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index f23081ea5..1f6090526 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -57,7 +57,7 @@ impl Multipart where R: HttpRequest { return Err(req); } - let boundary = req.get_boundary().unwrap().into(); + let boundary = req.get_boundary().unwrap().to_owned(); debug!("Boundary: {}", boundary); diff --git a/multipart/tests/multipart_server.rs b/multipart/tests/multipart_server.rs deleted file mode 100644 index b808ced2c..000000000 --- a/multipart/tests/multipart_server.rs +++ /dev/null @@ -1,52 +0,0 @@ -#![allow(dead_code)] - -extern crate hyper; -extern crate multipart; - -use hyper::server::{Listening, Server, Request, Response}; -use hyper::client::Request as ClientReq; -use hyper::status::StatusCode; -use hyper::Url; - -use multipart::server::Multipart; -use multipart::client::Multipart as ClientMulti; - -use std::old_io::net::ip::Ipv4Addr; - -use std::rand::random; - -fn ok_serv(req: Request, mut res: Response) { - let mut multipart = Multipart::from_request(req).ok().expect("Could not create multipart!"); - - multipart.foreach_entry(|&: _, _| ()); - - *res.status_mut() = StatusCode::Ok; - - res.start().unwrap().end().unwrap(); -} - -thread_local!(static PORT: u16 = random()); - -fn server() -> Listening { - let server = PORT.with(|port| Server::http(Ipv4Addr(127, 0, 0, 1), *port)); - server.listen(ok_serv).unwrap() -} - -#[test] -fn client_api_test() { - let mut server = server(); - - let address = PORT.with(|port| format!("http://localhost:{}/", port)); - - let request = ClientReq::post(Url::parse(&*address).unwrap()).unwrap(); - - let mut multipart = ClientMulti::new(); - - multipart.add_text("hello", "world"); - multipart.add_text("goodnight", "sun"); - multipart.sized = true; - - multipart.send(request).unwrap(); - - server.close().unwrap(); -} From b608e34ad7fd6227389ecd0f7bb8572683bc2cfa Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Wed, 8 Jul 2015 11:20:18 -0700 Subject: [PATCH 058/453] Clean up iterator expression --- multipart/src/server/boundary.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/multipart/src/server/boundary.rs b/multipart/src/server/boundary.rs index da2d1d423..74aae6b50 100644 --- a/multipart/src/server/boundary.rs +++ b/multipart/src/server/boundary.rs @@ -34,13 +34,13 @@ impl BoundaryReader where R: Read { if !self.boundary_read { let boundary_0 = self.boundary[0]; - // Style advice requested - for (search_idx, maybe_boundary) - in buf[self.search_idx..].windows(self.boundary.len()).enumerate() - .filter(|&(_, window)| window[0] == boundary_0) - { - self.boundary_read = self.boundary == maybe_boundary; - self.search_idx += search_idx; + let lookahead_iter = buf[self.search_idx..].windows(self.boundary.len()).enumerate(); + + for (search_idx, maybe_boundary) in lookahead_iter { + if maybe_boundary[0] == self.boundary[0] { + self.boundary_read = self.boundary == maybe_boundary; + self.search_idx += search_idx; + } } } From 538a73275cd6083fec24b89432faf837cf140c00 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Wed, 8 Jul 2015 11:39:19 -0700 Subject: [PATCH 059/453] Fix test, add env_logger, remove rustc-serialize --- multipart/Cargo.toml | 2 +- multipart/src/lib.rs | 2 +- multipart/src/server/boundary.rs | 17 +++++++++++------ 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index f08390ec0..78dec5d6e 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -16,10 +16,10 @@ keywords = ["form-data", "hyper", "http", "post", "upload"] [dependencies] log = "*" +env_logger = "*" mime = "*" mime_guess = "*" rand = "*" -rustc-serialize = "*" [dependencies.hyper] version = "*" diff --git a/multipart/src/lib.rs b/multipart/src/lib.rs index fc68eb0e6..4c0666809 100644 --- a/multipart/src/lib.rs +++ b/multipart/src/lib.rs @@ -1,9 +1,9 @@ #[macro_use] extern crate log; +extern crate env_logger; extern crate mime; extern crate mime_guess; extern crate rand; -extern crate rustc_serialize; #[cfg(feature = "hyper")] extern crate hyper; diff --git a/multipart/src/server/boundary.rs b/multipart/src/server/boundary.rs index 74aae6b50..3f10b5ccd 100644 --- a/multipart/src/server/boundary.rs +++ b/multipart/src/server/boundary.rs @@ -39,11 +39,15 @@ impl BoundaryReader where R: Read { for (search_idx, maybe_boundary) in lookahead_iter { if maybe_boundary[0] == self.boundary[0] { self.boundary_read = self.boundary == maybe_boundary; - self.search_idx += search_idx; + self.search_idx = search_idx; + + if self.boundary_read { + break; + } } } } - + debug!("Buf len: {} Search idx: {}", buf.len(), self.search_idx); Ok(&buf[..self.search_idx]) } @@ -63,8 +67,7 @@ impl BoundaryReader where R: Read { self.buffer.consume(consume_amt); self.search_idx = 0; self.boundary_read = false; - - Ok(()) +Ok(()) } #[allow(unused)] @@ -124,7 +127,7 @@ fn copy_bytes(src: &[u8], dst: &mut [u8]) { #[test] fn test_boundary() { use std::io::BufReader; - + const BOUNDARY: &'static str = "--boundary\r\n"; const TEST_VAL: &'static str = "\r --boundary\r @@ -134,6 +137,8 @@ dashed-value-2\r --boundary\r "; + ::env_logger::init().unwrap(); + let src = &mut TEST_VAL.as_bytes(); let mut reader = BoundaryReader::from_reader(src, BOUNDARY); @@ -141,7 +146,7 @@ dashed-value-2\r debug!("Read 1"); let _ = reader.read_to_string(buf).unwrap(); - debug!("{}", buf); + debug!("Buf: {:?}", buf); assert!(buf.trim().is_empty()); buf.clear(); From 0a06e52c05b9dfe63b3c87e097bb12f920a26eec Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Wed, 8 Jul 2015 12:27:01 -0700 Subject: [PATCH 060/453] Cleanup and doc fixes --- multipart/src/client/hyper.rs | 13 ++++++++++--- multipart/src/client/mod.rs | 23 ++++++++++++----------- multipart/src/server/boundary.rs | 8 ++------ multipart/src/server/mod.rs | 6 ++---- 4 files changed, 26 insertions(+), 24 deletions(-) diff --git a/multipart/src/client/hyper.rs b/multipart/src/client/hyper.rs index dd7323f3e..3a0cd7846 100644 --- a/multipart/src/client/hyper.rs +++ b/multipart/src/client/hyper.rs @@ -10,7 +10,7 @@ use super::{HttpRequest, HttpStream}; use std::io; use std::io::prelude::*; -impl super::HttpRequest for Request { +impl HttpRequest for Request { type Stream = Request; type Response = Response; type Error = HyperError; @@ -18,7 +18,6 @@ impl super::HttpRequest for Request { /// #Panics /// If the `Request` method is not `Method::Post`. fn apply_headers(&mut self, boundary: &str, content_len: Option) { - assert!(self.method() == Method::Post, "Multipart request must use POST method!"); let headers = self.headers_mut(); @@ -40,6 +39,14 @@ impl super::HttpRequest for Request { } } -impl super::HttpStream for Request { +impl HttpStream for Request { } + +fn multipart_mime(bound: &str) -> Mime { + Mime( + TopLevel::Multipart, SubLevel::Ext("form-data".into()), + vec![(Attr::Ext("boundary".into()), Value::Ext(bound.into()))] + ) +} + diff --git a/multipart/src/client/mod.rs b/multipart/src/client/mod.rs index 2617a3d49..050b3d32d 100644 --- a/multipart/src/client/mod.rs +++ b/multipart/src/client/mod.rs @@ -18,10 +18,9 @@ //! server doesn't support chunked requests or otherwise rejects them. //! //! [chunked-example]: http://en.wikipedia.org/wiki/Chunked_transfer_encoding#Example -use mime::{Mime, TopLevel, SubLevel, Attr, Value}; +use mime::Mime; use std::borrow::{Borrow, BorrowMut}; -use std::cell::Cell; use std::convert::AsRef; use std::fs::File; @@ -85,6 +84,9 @@ impl Multipart { /// Write a text field to this multipart request. /// `name` and `val` can be either owned `String` or `&str`. + /// + /// ##Errors + /// If something went wrong with the HTTP stream. pub fn write_text, V: Borrow>(mut self, name: N, val: V) -> Self { if self.last_err.is_none() { self.last_err = chain_result! { @@ -102,6 +104,10 @@ impl Multipart { /// /// If you want to set these values manually, or use another type that implements `Read`, /// use `.write_stream()`. + /// + /// ##Errors + /// If there was a problem opening the file (was a directory or didn't exist), + /// or if something went wrong with the HTTP stream. // Remove `File::path()`, GG Rust-lang pub fn write_file, P: AsRef>(mut self, name: N, path: P) -> Self { if self.last_err.is_none() { @@ -133,6 +139,9 @@ impl Multipart { /// multipart data has to be written to an in-memory buffer so its size can be calculated. /// /// Use `Read::take` if you wish to send data from a `Read` that will never end otherwise. + /// + /// ##Errors + /// If the reader returned an error, or if something went wrong with the HTTP stream. // RFC: How to format this declaration? pub fn write_stream, St: Read, St_: BorrowMut>( mut self, name: N, mut read: St_, filename: Option<&str>, content_type: Option @@ -170,7 +179,7 @@ impl Multipart { write!(self.stream, "{}\r\n", self.boundary) } - /// Finalize the request and return the response from the server. + /// Finalize the request and return the response from the server, or the last error if set. pub fn send(self) -> Result { match self.last_err { None => self.stream.finish(), @@ -187,14 +196,6 @@ where ::Error: From { } } - -fn multipart_mime(bound: &str) -> Mime { - Mime( - TopLevel::Multipart, SubLevel::Ext("form-data".into()), - vec![(Attr::Ext("boundary".into()), Value::Ext(bound.into()))] - ) -} - pub trait HttpRequest { type Stream: HttpStream; type Error: From + Into<::Error>; diff --git a/multipart/src/server/boundary.rs b/multipart/src/server/boundary.rs index 3f10b5ccd..a625b574c 100644 --- a/multipart/src/server/boundary.rs +++ b/multipart/src/server/boundary.rs @@ -16,7 +16,6 @@ pub struct BoundaryReader { boundary_read: bool, } - impl BoundaryReader where R: Read { #[doc(hidden)] pub fn from_reader>>(reader: R, boundary: B) -> BoundaryReader { @@ -32,8 +31,6 @@ impl BoundaryReader where R: Read { let buf = try!(self.buffer.fill_buf()); if !self.boundary_read { - let boundary_0 = self.boundary[0]; - let lookahead_iter = buf[self.search_idx..].windows(self.boundary.len()).enumerate(); for (search_idx, maybe_boundary) in lookahead_iter { @@ -70,6 +67,7 @@ impl BoundaryReader where R: Read { Ok(()) } + // Keeping this around to support nested boundaries later. #[allow(unused)] #[doc(hidden)] pub fn set_boundary>>(&mut self, boundary: B) { @@ -125,9 +123,7 @@ fn copy_bytes(src: &[u8], dst: &mut [u8]) { } #[test] -fn test_boundary() { - use std::io::BufReader; - +fn test_boundary() { const BOUNDARY: &'static str = "--boundary\r\n"; const TEST_VAL: &'static str = "\r --boundary\r diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index 1f6090526..227df8dbe 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -5,14 +5,12 @@ //! //! See the `Multipart` struct for more info. -use mime::{Mime, TopLevel, SubLevel, Attr, Value}; +use mime::Mime; use std::borrow::Borrow; -use std::cmp; -use std::collections::HashMap; use std::ops::Deref; -use std::fmt; +use std::collections::HashMap; use std::fs::{self, File}; use std::io; From aee3576076613edb1089b5e4d562986bd91a38ec Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Fri, 10 Jul 2015 15:29:42 -0700 Subject: [PATCH 061/453] Server API rework (may not compile) --- multipart/src/server/mod.rs | 296 +++++++++++++++++++++++------------- 1 file changed, 188 insertions(+), 108 deletions(-) diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index 227df8dbe..c7bd0cd16 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -26,6 +26,36 @@ mod boundary; #[cfg(feature = "hyper")] pub mod hyper; +macro_rules! try_opt ( + ($expr:expr) => ( + match $expr { + Some(val) => Some(val), + None => return None, + } + ) +); + +#[derive(Clone, Debug)] +pub enum MultipartError { + EndOfStream, + Io(io::Error), +} + +impl From for MultipartError { + fn from(err: io::Error) -> MultipartError { + MultipartError::Io(err) + } +} + +impl MultipartError { + fn is_end_of_stream(&self) -> bool { + match *self { + MultipartError::EndOfStream => true, + _ => false, + } + } +} + /// The server-side implementation of `multipart/form-data` requests. /// /// Create this with `Multipart::from_request()` passing a `server::Request` object from Hyper, @@ -39,12 +69,6 @@ pub struct Multipart { line_buf: String, } -macro_rules! try_find( - ($needle:expr, $haystack:expr, $err:expr) => ( - try!($haystack.find($needle).ok_or_else(|| line_error($err, $haystack))) - ) -); - impl Multipart where R: HttpRequest { /// If the given `HttpRequest` is a POST request of `Content-Type: multipart/form-data`, /// return the wrapped request as `Ok(Multipart)`, otherwise `Err(HttpRequest)`. @@ -75,32 +99,13 @@ impl Multipart where R: HttpRequest { /// ##Warning /// If the last returned entry had contents of type `MultipartField::File`, /// calling this again will discard any unread contents of that entry! - pub fn read_entry(&mut self) -> io::Result<(String, MultipartField)> { - try!(self.source.consume_boundary()); - let (disp_type, field_name, filename) = try!(self.read_content_disposition()); - - if disp_type != "form-data" { - return Err(io::Error::new( - io::ErrorKind::Other, - format!("Content-Disposition value: {:?} expected: \"form-data\"", disp_type), - )); - } + pub fn read_entry(&mut self) -> io::Result>> { + + } - if let Some(content_type) = try!(self.read_content_type()) { - let _ = try!(self.read_line()); // Consume empty line - Ok((field_name, - MultipartField::File( - MultipartFile::from_reader(filename, &mut self.source, &content_type) - ) - )) - } else { - // Empty line consumed by read_content_type() - let text = try!(self.read_to_string()); - // The last two characters are "\r\n". - // We can't do a simple trim because the content might be terminated - // with line separators we want to preserve. - Ok((field_name, MultipartField::Text(text[..text.len() - 2].into()))) - } + fn read_content_disposition(&mut self) -> io::Result> { + let line = try!(self.read_line()); + Ok(ContentDisp::read_from(line)) } /// Call `f` for each entry in the multipart request. @@ -108,68 +113,22 @@ impl Multipart where R: HttpRequest { /// since `Iterator::next()` can't use bound lifetimes. /// /// See https://www.reddit.com/r/rust/comments/2lkk\4\isize/concrete_lifetime_vs_bound_lifetime/ - pub fn foreach_entry(&mut self, mut foreach: F) where F: FnMut(String, MultipartField) { + /// + /// Returns `Ok(())` when all fields have been read, or the first error. + pub fn foreach_entry(&mut self, mut foreach: F) -> io::Result<()> where F: FnMut(MultipartField) { loop { match self.read_entry() { - Ok((name, field)) => foreach(name, field), - Err(err) => { - error!("Error reading Multipart: {}", err); - break; - }, + Ok(Some(field)) => foreach(field), + Ok(None) => return Ok(()), + Err(err) => return Err(err), } } } - fn read_content_disposition(&mut self) -> io::Result<(String, String, Option)> { - let line = try!(self.read_line()); - - // Find the end of CONT_DISP in the line - let disp_type = { - const CONT_DISP: &'static str = "Content-Disposition:"; - - let disp_idx = try_find!(CONT_DISP, &line, "Content-Disposition subheader not found!") - + CONT_DISP.len(); - - let disp_type_end = try_find!( - ';', &line[disp_idx..], - "Error parsing Content-Disposition value!" - ); - - line[disp_idx .. disp_idx + disp_type_end].trim().to_owned() - }; - - let field_name = { - const NAME: &'static str = "name=\""; - - let name_idx = try_find!(NAME, &line, "Error parsing field name!") + NAME.len(); - let name_end = try_find!('"', &line[name_idx ..], "Error parsing field name!"); - - line[name_idx .. name_idx + name_end].to_owned() // No trim here since it's in quotes. - }; - - let filename = { - const FILENAME: &'static str = "filename=\""; - - let filename_idx = line.find(FILENAME).map(|idx| idx + FILENAME.len()); - let filename_idxs = with(filename_idx, |&start| line[start ..].find('"')); - - filename_idxs.map(|(start, end)| line[start .. start + end].to_owned()) - }; - - Ok((disp_type, field_name, filename)) - } - - fn read_content_type(&mut self) -> io::Result> { + fn read_content_type<'a>(&'a mut self) -> io::Result> { debug!("Read content type!"); let line = try!(self.read_line()); - - const CONTENT_TYPE: &'static str = "Content-Type:"; - - let type_idx = line.find(CONTENT_TYPE); - - // FIXME Will not properly parse for multiple files! - // Does not expect boundary= - Ok(type_idx.map(|start| line[(start + CONTENT_TYPE.len())..].trim().to_owned())) + Ok(ContentType::read_from(line)) } /// Read the request fully, parsing all fields and saving all files @@ -199,6 +158,8 @@ impl Multipart where R: HttpRequest { } fn read_line(&mut self) -> io::Result<&str> { + self.line_buf.clear(); + match self.source.read_line(&mut self.line_buf) { Ok(read) => Ok(&self.line_buf[..read]), Err(err) => Err(err), @@ -206,6 +167,8 @@ impl Multipart where R: HttpRequest { } fn read_to_string(&mut self) -> io::Result<&str> { + self.line_buf.clear(); + match self.source.read_to_string(&mut self.line_buf) { Ok(read) => Ok(&self.line_buf[..read]), Err(err) => Err(err), @@ -235,35 +198,137 @@ fn line_error(msg: &str, line: &str) -> io::Error { ) } -/// A result of `Multipart::save_all()`. -pub struct Entries { - pub fields: HashMap, - pub files: HashMap, - /// The directory the files were saved under. - pub dir: PathBuf, +struct ContentType<'a> { + cont_type: &'a str, + boundary: Option<&'a str>, } -impl Entries { - fn with_path>(path: P) -> Entries { - Entries { - fields: HashMap::new(), - files: HashMap::new(), - dir: path.into(), +impl<'a> ContentType<'a> { + fn read_from(line: &'a str) -> Option> { + const CONTENT_TYPE: &'static str = "Content-Type:"; + const BOUNDARY: &'static str = "boundary=\""; + + let cont_type = get_str_after(CONTENT_TYPE, ';', line); + + if let Some((cont_type, after_cont_type)) = get_str_after(CONTENT_TYPE, ';', line) { + let boundary = get_str_after(BOUNDARY, '"', after_cont_type).map(|tup| tup.0); + + Some(ContentType { + cont_type: cont_type, + boundary: boundary, + }) + } else { + get_remainder_after(CONTENT_TYPE, line).map(|cont_type| + ContentType { cont_type: cont_type, boundary: None } + ) + } + } +} + +struct ContentDisp { + field_name: String, + filename: Option, +} + +impl ContentDisp { + fn read_from(line: &str) -> Option> { + debug!("Reading Content-Disposition from line: {:?}", line); + + if line.is_empty() { + return None; } + + const CONT_DISP: &'static str = "Content-Disposition:"; + const NAME: &'static str = "name=\""; + const FILENAME: &'static str = "filename=\""; + + let after_disp_type = { + let (disp_type, after_disp_type) = get_str_after(CONT_DISP, ';', line); + + if disp_type.trim() != "form-data" { + return None; + } + + after_disp_type + }; + + let (field_name, after_field_name) = try_opt!(get_str_after(NAME, '"', after_disp_type)); + + let filename = get_str_after(FILENAME, '"', after_field_name) + .map(|(filename, _)| filename.to_owned()); + + Some(ContentDisp { field_name: field_name.to_owned(), filename: filename }) } } +/// Get the string after `needle` in `haystack`, stopping before `end_val_delim` +fn get_str_after<'a>(needle: &str, end_val_delim: char, haystack: &'a str) -> Option<(&'a str, &'a str)> { + let val_start_idx = try_opt!(haystack.find(needle)) + needle.len(); + let val_end_idx = try_opt!(haystack[val_start_idx..].find(end_val_delim)); + Some(&haystack[val_start_idx..val_end_idx], &haystack[val_end_idx..]) +} + +/// Get everything after `needle` in `haystack` +fn get_remainder_after<'a>(needle: &str, haystack: &'a str) -> Option<(&'a str)> { + let val_start_idx = try_opt!(haystack.find(needle)) + needle.len(); + Some(&haystack[val_start_idx..]) +} + pub trait HttpRequest: Read { fn is_multipart(&self) -> bool; fn get_boundary(&self) -> Option<&str>; } -/// A field in a `multipart/form-data` request. -/// -/// This enum does not include the names of the fields, as those are yielded separately -/// by `server::Multipart::read_entry()`. +pub struct MultipartField<'a, R: 'a> { + /// The field's name from the form + pub name: String, + /// The data of the field. Can be text or binary. + pub data: MultipartData<'a, R>, +} + +impl<'a, R: HttpRequest + 'a> MultipartField<'a, R> { + fn read_from(multipart: &'a mut Multipart) -> io::Result>> { + try!(multipart.source.consume_boundary()); + + let cont_disp = match multipart.read_content_disposition() { + Ok(Some(cont_disp)) => cont_disp, + Ok(None) => return None, + Err(err) => return Err(err), + }; + + let data = match try!(multipart.read_content_type()) { + Some(content_type) => { + let _ = try!(multipart.read_line()); // Consume empty line + MultipartData::File( + MultipartFile::from_reader( + cont_disp.filename, + &mut multipart.source, + &content_type, + ) + ) + }, + None => { + // Empty line consumed by read_content_type() + let text = try!(multipart.read_to_string()); + // The last two characters are "\r\n". + // We can't do a simple trim because the content might be terminated + // with line separators we want to preserve. + MultipartData::Text(&text[..text.len() - 2]) + }, + }; + + Ok(Some( + MultipartField { + name: cont_disp.name, + data: data, + } + )) + } +} + +/// The data of a field in a `multipart/form-data` request. #[derive(Debug)] -pub enum MultipartField<'a, R: 'a> { +pub enum MultipartData<'a, R: 'a> { /// A text field. Text(&'a str), /// A file field, including the content type and optional filename @@ -272,11 +337,11 @@ pub enum MultipartField<'a, R: 'a> { // MultiFiles(Vec), /* TODO: Multiple files */ } -impl<'a, R> MultipartField<'a, R> { +impl<'a, R> MultipartData<'a, R> { /// Borrow this field as a text field, if possible. pub fn as_text(&self) -> Option<&str> { match *self { - MultipartField::Text(ref s) => Some(s), + MultipartData::Text(ref s) => Some(s), _ => None, } } @@ -285,7 +350,7 @@ impl<'a, R> MultipartField<'a, R> { /// Mutably borrows so the contents can be read. pub fn as_file(&mut self) -> Option<&mut MultipartFile<'a, R>> { match *self { - MultipartField::File(ref mut file) => Some(file), + MultipartData::File(ref mut file) => Some(file), _ => None, } } @@ -300,8 +365,8 @@ impl<'a, R> MultipartField<'a, R> { /// to save it to disk. #[derive(Debug)] pub struct MultipartFile<'a, R: 'a> { - filename: Option, - content_type: Mime, + pub filename: Option, + pub content_type: Mime, reader: &'a mut BoundaryReader, } @@ -370,6 +435,21 @@ impl<'a, R: Read> Read for MultipartFile<'a, R> { } } +/// A result of `Multipart::save_all()`. +pub struct Entries { + pub fields: HashMap, + pub files: HashMap>, + /// The directory the files were saved under. + pub dir: PathBuf, +} - +impl Entries { + fn with_path>(path: P) -> Entries { + Entries { + fields: HashMap::new(), + files: HashMap::new(), + dir: path.into(), + } + } +} From 0753b8b18e3b085de0ca0109d709a6cb0aa716d7 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sat, 11 Jul 2015 03:53:25 -0700 Subject: [PATCH 062/453] Finish refactoring server API --- multipart/src/lib.rs | 7 +++ multipart/src/server/mod.rs | 122 +++++++++++++++--------------------- 2 files changed, 57 insertions(+), 72 deletions(-) diff --git a/multipart/src/lib.rs b/multipart/src/lib.rs index 4c0666809..fb94725b7 100644 --- a/multipart/src/lib.rs +++ b/multipart/src/lib.rs @@ -10,6 +10,8 @@ extern crate hyper; use rand::Rng; +use std::path::PathBuf; + macro_rules! try_all { ($first_expr:expr, $($try_expr:expr),*) => ( try!($first_expr $(.and_then(|_| $try_expr))*); @@ -26,6 +28,11 @@ pub mod client; pub mod server; const BOUNDARY_LEN: usize = 16; +const DIRNAME_LEN: usize = 12; + +fn temp_dir() -> PathBuf { + random_alphanumeric(DIRNAME_LEN).into() +} fn random_alphanumeric(len: usize) -> String { rand::thread_rng().gen_ascii_chars().take(len).collect() diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index c7bd0cd16..db6322522 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -29,33 +29,12 @@ pub mod hyper; macro_rules! try_opt ( ($expr:expr) => ( match $expr { - Some(val) => Some(val), + Some(val) => val, None => return None, } ) ); -#[derive(Clone, Debug)] -pub enum MultipartError { - EndOfStream, - Io(io::Error), -} - -impl From for MultipartError { - fn from(err: io::Error) -> MultipartError { - MultipartError::Io(err) - } -} - -impl MultipartError { - fn is_end_of_stream(&self) -> bool { - match *self { - MultipartError::EndOfStream => true, - _ => false, - } - } -} - /// The server-side implementation of `multipart/form-data` requests. /// /// Create this with `Multipart::from_request()` passing a `server::Request` object from Hyper, @@ -100,7 +79,7 @@ impl Multipart where R: HttpRequest { /// If the last returned entry had contents of type `MultipartField::File`, /// calling this again will discard any unread contents of that entry! pub fn read_entry(&mut self) -> io::Result>> { - + MultipartField::read_from(self) } fn read_content_disposition(&mut self) -> io::Result> { @@ -125,36 +104,38 @@ impl Multipart where R: HttpRequest { } } - fn read_content_type<'a>(&'a mut self) -> io::Result> { + fn read_content_type(&mut self) -> io::Result> { debug!("Read content type!"); let line = try!(self.read_line()); Ok(ContentType::read_from(line)) } /// Read the request fully, parsing all fields and saving all files - /// to the given directory (if given) and return the result. + /// to the given directory or a random directory under `std::os::temp_dir()` + /// and return the result. /// - /// If `dir` is none, uses `std::os::tmpdir()`. - pub fn save_all(mut self, dir: Option<&Path>) -> io::Result { - let dir = dir.map_or_else(::std::env::temp_dir, |path| path.to_owned()); - + /// If there is an error in reading the request, returns the result so far along with the + /// error. + pub fn save_all(&mut self, dir: Option<&Path>) -> (Entries, Option) { + let dir = dir.map_or_else(::temp_dir, |path| path.to_owned()); let mut entries = Entries::with_path(dir); loop { match self.read_entry() { - Ok((name, MultipartField::Text(text))) => { entries.fields.insert(name, text.to_owned()); }, - Ok((name, MultipartField::File(mut file))) => { - let path = try!(file.save_in(&entries.dir)); - entries.files.insert(name, path); - }, - Err(err) => { - error!("Error reading Multipart: {}", err); - break; + Ok(Some(field)) => match field.data { + MultipartData::File(mut file) => { + entries.files.insert(field.name, file.save_in(&entries.dir)); + }, + MultipartData::Text(text) => { + entries.fields.insert(field.name, text.into()); + }, }, + Ok(None) => break, + Err(err) => return (entries, Some(err)), } } - Ok(entries) + (entries, None) } fn read_line(&mut self) -> io::Result<&str> { @@ -183,55 +164,48 @@ impl Deref for Multipart where R: HttpRequest { } } -fn with Option>(left: Option, right: F) -> Option<(T, U)> { - let temp = left.as_ref().and_then(right); - match (left, temp) { - (Some(lval), Some(rval)) => Some((lval, rval)), - _ => None, - } -} - -fn line_error(msg: &str, line: &str) -> io::Error { - io::Error::new( - io::ErrorKind::Other, - format!("Error: {:?} on line of request: {:?}", msg, line) - ) +struct ContentType { + cont_type: Mime, + #[allow(dead_code)] + boundary: Option, } -struct ContentType<'a> { - cont_type: &'a str, - boundary: Option<&'a str>, -} - -impl<'a> ContentType<'a> { - fn read_from(line: &'a str) -> Option> { +impl ContentType { + fn read_from(line: &str) -> Option { const CONTENT_TYPE: &'static str = "Content-Type:"; const BOUNDARY: &'static str = "boundary=\""; - let cont_type = get_str_after(CONTENT_TYPE, ';', line); + debug!("Reading Content-Type header from line: {:?}", line); if let Some((cont_type, after_cont_type)) = get_str_after(CONTENT_TYPE, ';', line) { - let boundary = get_str_after(BOUNDARY, '"', after_cont_type).map(|tup| tup.0); + let cont_type = read_content_type(cont_type); + + let boundary = get_str_after(BOUNDARY, '"', after_cont_type).map(|tup| tup.0.into()); Some(ContentType { cont_type: cont_type, boundary: boundary, }) } else { - get_remainder_after(CONTENT_TYPE, line).map(|cont_type| + get_remainder_after(CONTENT_TYPE, line).map(|cont_type| { + let cont_type = read_content_type(cont_type); ContentType { cont_type: cont_type, boundary: None } - ) + }) } } } +fn read_content_type(cont_type: &str) -> Mime { + cont_type.parse().ok().unwrap_or_else(::mime_guess::octet_stream) +} + struct ContentDisp { field_name: String, filename: Option, } impl ContentDisp { - fn read_from(line: &str) -> Option> { + fn read_from(line: &str) -> Option { debug!("Reading Content-Disposition from line: {:?}", line); if line.is_empty() { @@ -243,9 +217,11 @@ impl ContentDisp { const FILENAME: &'static str = "filename=\""; let after_disp_type = { - let (disp_type, after_disp_type) = get_str_after(CONT_DISP, ';', line); + let (disp_type, after_disp_type) = try_opt!(get_str_after(CONT_DISP, ';', line)); + let disp_type = disp_type.trim(); - if disp_type.trim() != "form-data" { + if disp_type != "form-data" { + error!("Unexpected Content-Disposition value: {:?}", disp_type); return None; } @@ -265,7 +241,7 @@ impl ContentDisp { fn get_str_after<'a>(needle: &str, end_val_delim: char, haystack: &'a str) -> Option<(&'a str, &'a str)> { let val_start_idx = try_opt!(haystack.find(needle)) + needle.len(); let val_end_idx = try_opt!(haystack[val_start_idx..].find(end_val_delim)); - Some(&haystack[val_start_idx..val_end_idx], &haystack[val_end_idx..]) + Some((&haystack[val_start_idx..val_end_idx], &haystack[val_end_idx..])) } /// Get everything after `needle` in `haystack` @@ -287,12 +263,12 @@ pub struct MultipartField<'a, R: 'a> { } impl<'a, R: HttpRequest + 'a> MultipartField<'a, R> { - fn read_from(multipart: &'a mut Multipart) -> io::Result>> { + fn read_from(multipart: &'a mut Multipart) -> io::Result>> { try!(multipart.source.consume_boundary()); let cont_disp = match multipart.read_content_disposition() { Ok(Some(cont_disp)) => cont_disp, - Ok(None) => return None, + Ok(None) => return Ok(None), Err(err) => return Err(err), }; @@ -303,7 +279,7 @@ impl<'a, R: HttpRequest + 'a> MultipartField<'a, R> { MultipartFile::from_reader( cont_disp.filename, &mut multipart.source, - &content_type, + content_type.cont_type, ) ) }, @@ -319,7 +295,7 @@ impl<'a, R: HttpRequest + 'a> MultipartField<'a, R> { Ok(Some( MultipartField { - name: cont_disp.name, + name: cont_disp.field_name, data: data, } )) @@ -372,11 +348,11 @@ pub struct MultipartFile<'a, R: 'a> { impl<'a, R: Read> MultipartFile<'a, R> { fn from_reader( - filename: Option, reader: &'a mut BoundaryReader, cont_type: &str + filename: Option, reader: &'a mut BoundaryReader, cont_type: Mime, ) -> MultipartFile<'a, R> { MultipartFile { filename: filename, - content_type: cont_type.parse::().ok().unwrap_or_else(::mime_guess::octet_stream), + content_type: cont_type, reader: reader, } } @@ -437,7 +413,9 @@ impl<'a, R: Read> Read for MultipartFile<'a, R> { /// A result of `Multipart::save_all()`. pub struct Entries { + /// The text files of the multipart request pub fields: HashMap, + /// A list of file field names and their save results. pub files: HashMap>, /// The directory the files were saved under. pub dir: PathBuf, From 22137a69fbe149291b8606c021c9ed78d0ffff84 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sun, 12 Jul 2015 14:52:54 -0700 Subject: [PATCH 063/453] Compiles and passes tests --- multipart/src/client/mod.rs | 34 ++++++++-- multipart/src/server/mod.rs | 7 +- multipart/tests/local.rs | 128 ++++++++++++++++++++++++++++++++++++ 3 files changed, 161 insertions(+), 8 deletions(-) create mode 100644 multipart/tests/local.rs diff --git a/multipart/src/client/mod.rs b/multipart/src/client/mod.rs index 050b3d32d..196b2a979 100644 --- a/multipart/src/client/mod.rs +++ b/multipart/src/client/mod.rs @@ -49,9 +49,10 @@ pub struct Multipart { stream: S, boundary: String, last_err: Option, + data_written: bool, } -impl Multipart { +impl Multipart { /// Create a new `Multipart` to wrap a request. /// /// ## Returns Error @@ -65,7 +66,8 @@ impl Multipart { Ok(Multipart { stream: stream, boundary: boundary, - last_err: None + last_err: None, + data_written: false, }) } } @@ -177,12 +179,19 @@ impl Multipart { fn write_boundary(&mut self) -> io::Result<()> { write!(self.stream, "{}\r\n", self.boundary) + .map(|res| { self.data_written = true; res }) } /// Finalize the request and return the response from the server, or the last error if set. - pub fn send(self) -> Result { + pub fn send(mut self) -> Result { match self.last_err { - None => self.stream.finish(), + None => { + if self.data_written { + try!(self.stream.write(b"--")); + } + + self.stream.finish() + }, Some(err) => Err(err), } } @@ -192,7 +201,7 @@ impl Multipart> where ::Error: From { /// Create a new `Multipart` using the `SizedRequest` wrapper around `req`. pub fn from_request_sized(req: R) -> Result { - Multipart::>::from_request(SizedRequest::from_request(req)) + Multipart::from_request(SizedRequest::from_request(req)) } } @@ -218,3 +227,18 @@ pub trait HttpStream: Write { fn finish(self) -> Result; } +impl HttpRequest for () { + type Stream = io::Sink; + type Error = io::Error; + + fn apply_headers(&mut self, _: &str, _: Option) { } + fn open_stream(self) -> Result { Ok(io::sink()) } +} + +impl HttpStream for io::Sink { + type Request = (); + type Response = io::Empty; + type Error = io::Error; + + fn finish(self) -> Result { Ok(io::empty()) } +} diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index db6322522..ad1d15087 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -54,11 +54,11 @@ impl Multipart where R: HttpRequest { pub fn from_request(req: R) -> Result, R> { if !req.is_multipart() { return Err(req); } - if req.get_boundary().is_none() { + if req.boundary().is_none() { return Err(req); } - let boundary = req.get_boundary().unwrap().to_owned(); + let boundary = req.boundary().unwrap().to_owned(); debug!("Boundary: {}", boundary); @@ -252,9 +252,10 @@ fn get_remainder_after<'a>(needle: &str, haystack: &'a str) -> Option<(&'a str)> pub trait HttpRequest: Read { fn is_multipart(&self) -> bool; - fn get_boundary(&self) -> Option<&str>; + fn boundary(&self) -> Option<&str>; } +#[derive(Debug)] pub struct MultipartField<'a, R: 'a> { /// The field's name from the form pub name: String, diff --git a/multipart/tests/local.rs b/multipart/tests/local.rs new file mode 100644 index 000000000..70b3cf42c --- /dev/null +++ b/multipart/tests/local.rs @@ -0,0 +1,128 @@ +#[macro_use] +extern crate log; +extern crate env_logger; + +extern crate multipart; + +use multipart::client::HttpRequest as ClientRequest; +use multipart::client::HttpStream as ClientStream; + +use multipart::server::HttpRequest as ServerRequest; + +use std::io; +use std::io::prelude::*; + +#[test] +fn local_test() { + let buf = test_client(); + test_server(buf); +} + +fn test_client() -> HttpBuffer { + use multipart::client::Multipart; + + let request = MockClientRequest::default(); + + Multipart::from_request(request).unwrap() + .write_text("hello", "world") + .write_text("goodnight", "sun") + .send().unwrap() +} + +fn test_server(buf: HttpBuffer) { + use multipart::server::Multipart; + + let mut multipart = Multipart::from_request(buf.for_server()) + .unwrap_or_else(|_| panic!("Buffer should be multipart!")); + + while let Ok(Some(field)) = multipart.read_entry() { + match &*field.name { + "hello" => assert_eq!(field.data.as_text(), Some("world")), + "goodnight" => assert_eq!(field.data.as_text(), Some("sun")), + _ => panic!("Unexpected field: {:?}", field), + } + } +} + +#[derive(Default, Debug)] +struct MockClientRequest { + boundary: Option, + content_len: Option, +} + +impl ClientRequest for MockClientRequest { + type Stream = HttpBuffer; + type Error = io::Error; + + fn apply_headers(&mut self, boundary: &str, content_len: Option) { + self.boundary = Some(boundary.into()); + self.content_len = content_len; + } + + fn open_stream(self) -> Result { + debug!("MockClientRequest::open_stream called! {:?}", self); + let boundary = self.boundary.expect("HttpRequest::set_headers() was not called!"); + + Ok(HttpBuffer { buf: Vec::new(), boundary: boundary, content_len: self.content_len }) + } +} + +#[derive(Debug)] +struct HttpBuffer { + buf: Vec, + boundary: String, + content_len: Option, +} + +impl HttpBuffer { + fn for_server(&self) -> ServerBuffer { + ServerBuffer { + data: &self.buf, + boundary: &self.boundary, + content_len: self.content_len, + } + } +} + +impl Write for HttpBuffer { + fn write(&mut self, data: &[u8]) -> io::Result { + self.buf.write(data) + } + + fn flush(&mut self) -> io::Result<()> { + self.buf.flush() + } +} + +impl Read for HttpBuffer { + fn read(&mut self, _: &mut [u8]) -> io::Result { + unimplemented!() + } +} + +impl ClientStream for HttpBuffer { + type Request = MockClientRequest; + type Response = HttpBuffer; + type Error = io::Error; + + fn finish(self) -> Result { Ok(self) } +} + +#[derive(Debug)] +struct ServerBuffer<'a> { + data: &'a [u8], + boundary: &'a str, + content_len: Option, +} + +impl<'a> Read for ServerBuffer<'a> { + fn read(&mut self, out: &mut [u8]) -> io::Result { + self.data.read(out) + } +} + +impl<'a> ServerRequest for ServerBuffer<'a> { + fn is_multipart(&self) -> bool { true } + fn boundary(&self) -> Option<&str> { Some(&self.boundary) } +} + From 092e6a51dc6ff9648d92a57d3b550eccce0b8f60 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Tue, 14 Jul 2015 02:19:12 -0700 Subject: [PATCH 064/453] Fix some nigglings in API, add doc comments, Hyper support. --- multipart/src/client/hyper.rs | 44 +++--- multipart/src/client/mod.rs | 73 +++++----- multipart/src/client/sized.rs | 15 ++- multipart/src/lib.rs | 6 + multipart/src/server/boundary.rs | 2 +- multipart/src/server/hyper.rs | 97 +++++++------- multipart/src/server/mod.rs | 222 +++++++++++++++++++------------ multipart/tests/local.rs | 9 +- 8 files changed, 274 insertions(+), 194 deletions(-) diff --git a/multipart/src/client/hyper.rs b/multipart/src/client/hyper.rs index 3a0cd7846..0e4dff889 100644 --- a/multipart/src/client/hyper.rs +++ b/multipart/src/client/hyper.rs @@ -1,3 +1,7 @@ +//! Client-side integration with [Hyper](https://github.com/hyperium/hyper). +//! Enabled with the `hyper` feature. +//! +//! Contains `impl HttpRequest for Request` and `impl HttpStream for Request`. use hyper::client::request::Request; use hyper::client::response::Response; use hyper::error::Error as HyperError; @@ -5,42 +9,52 @@ use hyper::header::{ContentType, ContentLength}; use hyper::method::Method; use hyper::net::{Fresh, Streaming}; -use super::{HttpRequest, HttpStream}; +use mime::{Mime, TopLevel, SubLevel, Attr, Value}; -use std::io; -use std::io::prelude::*; +use super::{HttpRequest, HttpStream}; impl HttpRequest for Request { type Stream = Request; - type Response = Response; type Error = HyperError; /// #Panics - /// If the `Request` method is not `Method::Post`. - fn apply_headers(&mut self, boundary: &str, content_len: Option) { - assert!(self.method() == Method::Post, "Multipart request must use POST method!"); + /// If `self.method() != Method::Post`. + fn apply_headers(&mut self, boundary: &str, content_len: Option) -> bool { + if self.method() != Method::Post { + error!( + "Expected Hyper request method to be `Post`, was actually `{:?}`", + self.method() + ); + + return false; + } let headers = self.headers_mut(); - headers.set(ContentType(super::multipart_mime(boundary))); + headers.set(ContentType(multipart_mime(boundary))); if let Some(size) = content_len { headers.set(ContentLength(size)); } - debug!("Hyper headers: {}", self.headers()); + debug!("Hyper headers: {}", headers); + + true } - fn send(self, send_fn: F) -> Self::RequestResult - where F: FnOnce(&mut Request) -> io::Result<()> { - let mut req = try!(self.start()); - try!(send_fn(&mut req)); - req.send() + fn open_stream(self) -> Result { + self.start() } } impl HttpStream for Request { - + type Request = Request; + type Response = Response; + type Error = HyperError; + + fn finish(self) -> Result { + self.send() + } } fn multipart_mime(bound: &str) -> Mime { diff --git a/multipart/src/client/mod.rs b/multipart/src/client/mod.rs index 196b2a979..fd9e5194e 100644 --- a/multipart/src/client/mod.rs +++ b/multipart/src/client/mod.rs @@ -1,23 +1,6 @@ -//! The client side implementation of `multipart/form-data` requests. +//! The client-side abstraction for multipart requests. //! //! Use this when sending POST requests with files to a server. -//! -//! `ChunkedMultipart` sends chunked requests (recommended), -//! while `SizedMultipart` sends sized requests. -//! -//! Both implement the `MultipartRequest` trait for their core API. -//! -//! Sized requests are more human-readable and use less bandwidth -//! (as chunking adds [significant visual noise and overhead][chunked-example]), -//! but they must be able to load their entirety, including the contents of all files -//! and streams, into memory so the request body can be measured and its size set -//! in the `Content-Length` header. -//! -//! You should really only use sized requests if you intend to inspect the data manually on the -//! server side, as it will produce a more human-readable request body. Also, of course, if the -//! server doesn't support chunked requests or otherwise rejects them. -//! -//! [chunked-example]: http://en.wikipedia.org/wiki/Chunked_transfer_encoding#Example use mime::Mime; use std::borrow::{Borrow, BorrowMut}; @@ -30,10 +13,23 @@ use std::io::prelude::*; use std::path::Path; #[cfg(feature = "hyper")] -mod hyper; +pub mod hyper; mod sized; +/// A wrapper around a request object that measures the request body and sets the `Content-Length` +/// header to its size in bytes. +/// +/// Sized requests are more human-readable and use less bandwidth +/// (as chunking adds [visual noise and overhead][chunked-example]), +/// but they must be able to load their entirety, including the contents of all files +/// and streams, into memory so the request body can be measured. +/// +/// You should really only use sized requests if you intend to inspect the data manually on the +/// server side, as it will produce a more human-readable request body. Also, of course, if the +/// server doesn't support chunked requests or otherwise rejects them. +/// +/// [chunked-example]: http://en.wikipedia.org/wiki/Chunked_transfer_encoding#Example pub use self::sized::SizedRequest; @@ -110,7 +106,6 @@ impl Multipart { /// ##Errors /// If there was a problem opening the file (was a directory or didn't exist), /// or if something went wrong with the HTTP stream. - // Remove `File::path()`, GG Rust-lang pub fn write_file, P: AsRef>(mut self, name: N, path: P) -> Self { if self.last_err.is_none() { let path = path.as_ref(); @@ -130,7 +125,7 @@ impl Multipart { } /// Write a byte stream to the multipart request as a file field, supplying `filename` if given, - /// and `content_type` if given or `application/octet-stream` if not. + /// and `content_type` if given or `"application/octet-stream"` if not. /// /// ##Warning /// The given `Read` **must** be able to read to EOF (end of file/no more data), meaning @@ -140,7 +135,8 @@ impl Multipart { /// When using `sized::SizedRequest` this also can cause out-of-control memory usage as the /// multipart data has to be written to an in-memory buffer so its size can be calculated. /// - /// Use `Read::take` if you wish to send data from a `Read` that will never end otherwise. + /// Use `Read::take(usize)` if you wish to send data from a `Read` + /// that will never return EOF otherwise. /// /// ##Errors /// If the reader returned an error, or if something went wrong with the HTTP stream. @@ -187,6 +183,7 @@ impl Multipart { match self.last_err { None => { if self.data_written { + // Write two hyphens after the last boundary occurrence. try!(self.stream.write(b"--")); } @@ -205,25 +202,37 @@ where ::Error: From { } } +/// A trait describing an HTTP request that can be used to send multipart data. pub trait HttpRequest { - type Stream: HttpStream; + /// The HTTP stream type that can be opend by this request, to which the multipart data will be + /// written. + type Stream: HttpStream; + /// The error type for this request. + /// Must be compatible with `io::Error` as well as `Self::HttpStream::Error` type Error: From + Into<::Error>; - /// Set the `ContentType` header to `multipart/form-data` and supply the `boundary` value. + /// Set the `Content-Type` header to `multipart/form-data` and supply the `boundary` value. /// If `content_len` is given, set the `Content-Length` header to its value. - fn apply_headers(&mut self, boundary: &str, content_len: Option); + /// + /// Return `true` if any and all sanity checks passed and the stream is ready to be opened, + /// or `false` otherwise. + fn apply_headers(&mut self, boundary: &str, content_len: Option) -> bool; - /// Open the request stream and return it, after which point the request body will be - /// written. `HttpStream::finish()` will be called after the body has finished being written. + /// Open the request stream and return it or any error otherwise. fn open_stream(self) -> Result; } +/// A trait describing an open HTTP stream that can be written to. pub trait HttpStream: Write { + /// The request type that opened this stream. type Request: HttpRequest; - type Response: Read; + /// The response type that will be returned after the request is completed. + type Response; + /// The error type for this stream. + /// Must be compatible with `io::Error` as well as `Self::Request::Error`. type Error: From + From<::Error>; - /// Finalize and close the stream, returning the HTTP response object. + /// Finalize and close the stream and return the response object, or any error otherwise. fn finish(self) -> Result; } @@ -231,14 +240,14 @@ impl HttpRequest for () { type Stream = io::Sink; type Error = io::Error; - fn apply_headers(&mut self, _: &str, _: Option) { } + fn apply_headers(&mut self, _: &str, _: Option) -> bool { true } fn open_stream(self) -> Result { Ok(io::sink()) } } impl HttpStream for io::Sink { type Request = (); - type Response = io::Empty; + type Response = (); type Error = io::Error; - fn finish(self) -> Result { Ok(io::empty()) } + fn finish(self) -> Result { Ok(()) } } diff --git a/multipart/src/client/sized.rs b/multipart/src/client/sized.rs index 225c39b43..019a9b15a 100644 --- a/multipart/src/client/sized.rs +++ b/multipart/src/client/sized.rs @@ -38,9 +38,10 @@ where ::Error: From { type Error = R::Error; /// `SizedRequest` ignores `_content_len` because it sets its own later. - fn apply_headers(&mut self, boundary: &str, _content_len: Option) { + fn apply_headers(&mut self, boundary: &str, _content_len: Option) -> bool { self.boundary.clear(); self.boundary.push_str(boundary); + true } fn open_stream(mut self) -> Result { @@ -55,9 +56,15 @@ where ::Error: From { type Response = <::Stream as HttpStream>::Response; type Error = <::Stream as HttpStream>::Error; - fn finish(mut self) -> Result { - let content_len = self.buffer.len(); - self.inner.apply_headers(&self.boundary, Some(content_len)); + fn finish(mut self) -> Result { + let content_len = self.buffer.len() as u64; + + if !self.inner.apply_headers(&self.boundary, Some(content_len)) { + return Err(io::Error::new( + io::ErrorKind::Other, + "SizedRequest failed to apply headers to wrapped request." + ).into()); + } let mut req = try!(self.inner.open_stream()); try!(io::copy(&mut &self.buffer[..], &mut req)); diff --git a/multipart/src/lib.rs b/multipart/src/lib.rs index fb94725b7..cb65c61a7 100644 --- a/multipart/src/lib.rs +++ b/multipart/src/lib.rs @@ -1,3 +1,9 @@ +//! Client- and server-side abstractions for HTTP `multipart/form-data` requests. +//! +//! Features: +//! * `hyper`: Enable client- and server-side integration with +//! [Hyper](https:://github.com/hyperium/hyper) +#![warn(missing_docs)] #[macro_use] extern crate log; extern crate env_logger; diff --git a/multipart/src/server/boundary.rs b/multipart/src/server/boundary.rs index a625b574c..d146e1959 100644 --- a/multipart/src/server/boundary.rs +++ b/multipart/src/server/boundary.rs @@ -7,7 +7,7 @@ use std::io::prelude::*; use std::ptr; -/// A struct implementing `Read` that will yield bytes until it sees a given sequence. +/// A struct implementing `Read` and `BufRead` that will yield bytes until it sees a given sequence. #[derive(Debug)] pub struct BoundaryReader { buffer: BufReader, diff --git a/multipart/src/server/hyper.rs b/multipart/src/server/hyper.rs index a0fbe43cb..086714c76 100644 --- a/multipart/src/server/hyper.rs +++ b/multipart/src/server/hyper.rs @@ -1,16 +1,19 @@ -//! Convenient wrappers for `hyper::server::Handler` - +//! Server-side integration with [Hyper](https://github.com/hyperium/hyper). +//! Enabled with the `hyper` feature. +use hyper::net::Fresh; use hyper::header::ContentType; use hyper::method::Method; use hyper::server::{Handler, Request, Response}; -use super::Multipart; +use mime::{Mime, TopLevel, SubLevel, Attr, Value}; + +use super::{Multipart, HttpRequest}; /// A container that implements `hyper::server::Handler` which will switch /// the handler implementation depending on if the incoming request is multipart or not. /// /// Create an instance with `new()` and pass it to `hyper::server::Server::listen()` where -/// you would normally pass a `Handler` instance, usually a static function. +/// you would normally pass a `Handler` instance. /// /// A convenient wrapper for `Multipart::from_request()`. pub struct Switch { @@ -30,7 +33,7 @@ impl Switch where H: Handler, M: MultipartHandler { } impl Handler for Switch where H: Handler, M: MultipartHandler { - fn handle<'a>(&'a self, req: Request<'a>, res: Response<'a>) { + fn handle<'a, 'k>(&'a self, req: Request<'a, 'k>, res: Response<'a, Fresh>) { match Multipart::from_request(req) { Ok(multi) => self.multipart.handle_multipart(multi, res), Err(req) => self.normal.handle(req, res), @@ -40,64 +43,56 @@ impl Handler for Switch where H: Handler, M: MultipartHandler { /// A trait defining a type that can handle an incoming multipart request. /// -/// Extends to unboxed closures of the type `Fn(Multipart, Response)`, +/// Extends to closures of the type `Fn(Multipart, Response)`, /// and subsequently static functions. -/// -/// Since `Multipart` implements `Deref`, you can still access -/// the fields on `Request`, such as `Request::uri` or `Request::headers`. pub trait MultipartHandler: Send + Sync { /// Generate a response from this multipart request. - fn handle_multipart<'a>(&self, multipart: Multipart<'a>, response: Response); + fn handle_multipart<'a, 'k>(&self, + multipart: Multipart>, + response: Response<'a, Fresh>); } -impl MultipartHandler for F where F: for<'a> Fn(Multipart<'a>, Response) + Send + Sync { - fn handle_multipart<'a>(&self, multipart: Multipart<'a>, response: Response) { +impl MultipartHandler for F +where F: Fn(Multipart, Response), F: Send + Sync { + fn handle_multipart<'a, 'k>(&self, + multipart: Multipart>, + response: Response<'a, Fresh>) { (*self)(multipart, response); } } -/// A container for an unboxed closure that implements `hyper::server::Handler`. -/// -/// This exists because as of this writing, `Handler` is not automatically implemented for -/// compatible unboxed closures (though this will likely change). -/// -/// No private fields, instantiate directly. -pub struct UnboxedHandler { - /// The closure to call - pub f: F, -} - -impl Handler for UnboxedHandler where F: Fn(Request, Response) + Send + Sync { - fn handle(&self, req: Request, res: Response) { - (self.f)(req, res); - } -} - -fn is_multipart_formdata(req: &Request) -> bool { - req.method == Method::Post && req.headers.get::().map_or(false, |ct| { - let ContentType(ref mime) = *ct; +impl<'a, 'b> HttpRequest for Request<'a, 'b> { + fn is_multipart(&self) -> bool { + self.method == Method::Post && + self.headers.get::().map_or(false, |ct| { + let ContentType(ref mime) = *ct; - debug!("Content-Type: {}", mime); + debug!("Content-Type: {}", mime); - match *mime { - Mime(TopLevel::Multipart, SubLevel::FormData, _) => true, - _ => false, - } - }) -} + match *mime { + Mime(TopLevel::Multipart, SubLevel::FormData, _) => true, + _ => false, + } + }) + } -fn get_boundary(ct: &ContentType) -> Option { - let ContentType(ref mime) = *ct; - let Mime(_, _, ref params) = *mime; + fn boundary(&self) -> Option<&str> { + self.headers.get::().and_then(|ct| { + let ContentType(ref mime) = *ct; + let Mime(_, _, ref params) = *mime; - params.iter().find(|&&(ref name, _)| - if let Attr::Ext(ref name) = *name { - "boundary" == &**name - } else { false } - ).and_then(|&(_, ref val)| - if let Value::Ext(ref val) = *val { - Some(val.clone()) - } else { None } - ) + params.iter().find(|&&(ref name, _)| + match *name { + Attr::Ext(ref name) => "boundary" == name, + _ => false, + } + ).and_then(|&(_, ref val)| + match *val { + Value::Ext(ref val) => Some(&**val), + _ => None, + } + ) + }) + } } diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index ad1d15087..7ad7c7c28 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -1,14 +1,13 @@ -//! The server-side implementation of `multipart/form-data` requests. +//! The server-side abstraction for multipart requests requests. //! -//! Use this when you are implementing a server on top of Hyper and want to -//! to parse and serve POST `multipart/form-data` requests. +//! Use this when you are implementing an HTTP server and want to +//! to accept, parse, and serve HTTP `multipart/form-data` requests (file uploads). //! //! See the `Multipart` struct for more info. use mime::Mime; use std::borrow::Borrow; -use std::ops::Deref; use std::collections::HashMap; @@ -37,20 +36,19 @@ macro_rules! try_opt ( /// The server-side implementation of `multipart/form-data` requests. /// -/// Create this with `Multipart::from_request()` passing a `server::Request` object from Hyper, -/// or give Hyper a `handler::Switch` instance instead, -/// then read individual entries with `.read_entry()` or process them all at once with -/// `.foreach_entry()`. -/// -/// Implements `Deref` to allow access to read-only fields on `Request` without copying. +/// Implements `Borrow` to allow access to the request object. pub struct Multipart { source: BoundaryReader, line_buf: String, + /// The directory for saving files in this request. + /// By default, this is set to a subdirectory of `std::env::temp_dir()` with a + /// random alphanumeric name. + pub save_dir: PathBuf, } impl Multipart where R: HttpRequest { - /// If the given `HttpRequest` is a POST request of `Content-Type: multipart/form-data`, - /// return the wrapped request as `Ok(Multipart)`, otherwise `Err(HttpRequest)`. + /// If the given `R: HttpRequest` is a POST request of `Content-Type: multipart/form-data`, + /// return the wrapped request as `Ok(Multipart)`, otherwise `Err(R)`. pub fn from_request(req: R) -> Result, R> { if !req.is_multipart() { return Err(req); } @@ -66,18 +64,17 @@ impl Multipart where R: HttpRequest { Multipart { source: BoundaryReader::from_reader(req, boundary), line_buf: String::new(), + save_dir: ::temp_dir(), } ) } - /// Read an entry from this multipart request, returning a pair with the field's name and - /// contents. This will return an End of File error if there are no more entries. - /// - /// To get to the data, you will need to match on `MultipartField`. + /// Read the next entry from this multipart request, returning a struct with the field's name and + /// data. See `MultipartField` for more info. /// - /// ##Warning - /// If the last returned entry had contents of type `MultipartField::File`, - /// calling this again will discard any unread contents of that entry! + /// ##Warning: Risk of Data Loss + /// If the previously returned entry had contents of type `MultipartField::File`, + /// calling this again will discard any unread contents of that entry. pub fn read_entry(&mut self) -> io::Result>> { MultipartField::read_from(self) } @@ -88,10 +85,9 @@ impl Multipart where R: HttpRequest { } /// Call `f` for each entry in the multipart request. - /// This is a substitute for `Multipart` implementing `Iterator`, - /// since `Iterator::next()` can't use bound lifetimes. - /// - /// See https://www.reddit.com/r/rust/comments/2lkk\4\isize/concrete_lifetime_vs_bound_lifetime/ + /// + /// This is a substitute for Rust not supporting streaming iterators (where the return value + /// from `next()` borrows the iterator for a bound lifetime). /// /// Returns `Ok(())` when all fields have been read, or the first error. pub fn foreach_entry(&mut self, mut foreach: F) -> io::Result<()> where F: FnMut(MultipartField) { @@ -110,21 +106,18 @@ impl Multipart where R: HttpRequest { Ok(ContentType::read_from(line)) } - /// Read the request fully, parsing all fields and saving all files - /// to the given directory or a random directory under `std::os::temp_dir()` - /// and return the result. + /// Read the request fully, parsing all fields and saving all files in `self.save_dir`. /// - /// If there is an error in reading the request, returns the result so far along with the + /// If there is an error in reading the request, returns the partial result along with the /// error. - pub fn save_all(&mut self, dir: Option<&Path>) -> (Entries, Option) { - let dir = dir.map_or_else(::temp_dir, |path| path.to_owned()); - let mut entries = Entries::with_path(dir); + pub fn save_all(&mut self) -> (Entries, Option) { + let mut entries = Entries::with_path(self.save_dir.clone()); loop { match self.read_entry() { Ok(Some(field)) => match field.data { MultipartData::File(mut file) => { - entries.files.insert(field.name, file.save_in(&entries.dir)); + entries.files.insert(field.name, file.save()); }, MultipartData::Text(text) => { entries.fields.insert(field.name, text.into()); @@ -157,15 +150,14 @@ impl Multipart where R: HttpRequest { } } -impl Deref for Multipart where R: HttpRequest { - type Target = R; - fn deref(&self) -> &R { +impl Borrow for Multipart where R: HttpRequest { + fn borrow(&self) -> &R { self.source.borrow() } } struct ContentType { - cont_type: Mime, + val: Mime, #[allow(dead_code)] boundary: Option, } @@ -178,18 +170,18 @@ impl ContentType { debug!("Reading Content-Type header from line: {:?}", line); if let Some((cont_type, after_cont_type)) = get_str_after(CONTENT_TYPE, ';', line) { - let cont_type = read_content_type(cont_type); + let content_type = read_content_type(cont_type); let boundary = get_str_after(BOUNDARY, '"', after_cont_type).map(|tup| tup.0.into()); Some(ContentType { - cont_type: cont_type, + val: content_type, boundary: boundary, }) } else { get_remainder_after(CONTENT_TYPE, line).map(|cont_type| { - let cont_type = read_content_type(cont_type); - ContentType { cont_type: cont_type, boundary: None } + let content_type = read_content_type(cont_type); + ContentType { val: content_type, boundary: None } }) } } @@ -250,11 +242,15 @@ fn get_remainder_after<'a>(needle: &str, haystack: &'a str) -> Option<(&'a str)> Some(&haystack[val_start_idx..]) } +/// A server-side HTTP request that may or may not be multipart. pub trait HttpRequest: Read { + /// Return `true` if this request is a `multipart/form-data` request, `false` otherwise. fn is_multipart(&self) -> bool; + /// Get the boundary string of this request if it is `multipart/form-data`. fn boundary(&self) -> Option<&str>; } +/// A field in a multipart request. May be either text or a binary stream (file). #[derive(Debug)] pub struct MultipartField<'a, R: 'a> { /// The field's name from the form @@ -277,10 +273,11 @@ impl<'a, R: HttpRequest + 'a> MultipartField<'a, R> { Some(content_type) => { let _ = try!(multipart.read_line()); // Consume empty line MultipartData::File( - MultipartFile::from_reader( + MultipartFile::from_stream( cont_disp.filename, - &mut multipart.source, - content_type.cont_type, + content_type.val, + &multipart.save_dir, + &mut multipart.source, ) ) }, @@ -306,16 +303,16 @@ impl<'a, R: HttpRequest + 'a> MultipartField<'a, R> { /// The data of a field in a `multipart/form-data` request. #[derive(Debug)] pub enum MultipartData<'a, R: 'a> { - /// A text field. + /// The field's payload is a text string. Text(&'a str), - /// A file field, including the content type and optional filename - /// along with a `Read` implementation for getting the contents. + /// The field's payload is a binary stream (file). File(MultipartFile<'a, R>), - // MultiFiles(Vec), /* TODO: Multiple files */ + // TODO: Support multiple files per field (nested boundaries) + // MultiFiles(Vec), } impl<'a, R> MultipartData<'a, R> { - /// Borrow this field as a text field, if possible. + /// Borrow this payload as a text field, if possible. pub fn as_text(&self) -> Option<&str> { match *self { MultipartData::Text(ref s) => Some(s), @@ -323,7 +320,7 @@ impl<'a, R> MultipartData<'a, R> { } } - /// Borrow this field as a file field, if possible + /// Borrow this payload as a file field, if possible. /// Mutably borrows so the contents can be read. pub fn as_file(&mut self) -> Option<&mut MultipartFile<'a, R>> { match *self { @@ -336,87 +333,127 @@ impl<'a, R> MultipartData<'a, R> { /// A representation of a file in HTTP `multipart/form-data`. /// /// Note that the file is not yet saved to the system; -/// instead, the struct implements a `Reader` that points +/// instead, this struct exposes `Read` and `BufRead` impls which point /// to the beginning of the file's contents in the HTTP stream. +/// /// You can read it to EOF, or use one of the `save_*()` methods here /// to save it to disk. #[derive(Debug)] pub struct MultipartFile<'a, R: 'a> { - pub filename: Option, - pub content_type: Mime, - reader: &'a mut BoundaryReader, + filename: Option, + content_type: Mime, + save_dir: &'a Path, + stream: &'a mut BoundaryReader, } impl<'a, R: Read> MultipartFile<'a, R> { - fn from_reader( - filename: Option, reader: &'a mut BoundaryReader, cont_type: Mime, - ) -> MultipartFile<'a, R> { + fn from_stream(filename: Option, + content_type: Mime, + save_dir: &'a Path, + stream: &'a mut BoundaryReader) -> MultipartFile<'a, R> { MultipartFile { filename: filename, - content_type: cont_type, - reader: reader, + content_type: content_type, + save_dir: save_dir, + stream: stream, } } - /// Save this file to `path`, discarding the filename. + /// Save this file to `path`. /// - /// If successful, the file can be found at `path`. - pub fn save_as(&mut self, path: &Path) -> io::Result<()> { + /// Returns the number of bytes written on success, or any errors otherwise. + /// + /// Retries when `io::Error::kind() == io::ErrorKind::Interrupted`. + pub fn save_as(&mut self, path: &Path) -> io::Result { let mut file = try!(File::create(path)); - io::copy(self.reader, &mut file).and(Ok(())) + retry_on_interrupt(|| io::copy(self.stream, &mut file)) } - /// Save this file in the directory described by `dir`, - /// appending `filename` if present, or a random string otherwise. + /// Save this file in the directory pointed at by `dir`, + /// using `self.filename()` if present, or a random alphanumeric string otherwise. + /// + /// Any missing directories in the `dir` path will be created. /// - /// Returns the created file's path on success. + /// `self.filename()` is sanitized of all file separators before being appended to `dir`. /// - /// ###Panics - /// If `dir` does not represent a directory. + /// Returns the created file's path on success, or any errors otherwise. + /// + /// Retries when `io::Error::kind() == io::ErrorKind::Interrupted`. pub fn save_in(&mut self, dir: &Path) -> io::Result { - let meta = try!(fs::metadata(dir)); - assert!(meta.is_dir(), "Given path is not a directory!"); - - let path = dir.join(::random_alphanumeric(8)); - - try!(self.save_as(&path)); - - Ok(path) + try!(fs::create_dir_all(dir)); + let path = self.gen_safe_file_path(dir); + self.save_as(&path).map(move |_| path) } - /// Save this file in the OS temp directory, returned from `std::env::temp_dir()`. + /// Save this file in the directory pointed at by `self.save_dir`, + /// using `self.filename()` if present, or a random alphanumeric string otherwise. + /// + /// Any missing directories in the `self.save_dir` path will be created. /// - /// Returns the created file's path on success. - pub fn save_temp(&mut self) -> io::Result { - use std::env; - - self.save_in(&env::temp_dir()) + /// `self.filename()` is sanitized of all file separators before being appended to `self.save_dir`. + /// + /// Returns the created file's path on success, or any errors otherwise. + /// + /// Retries when `io::Error::kind() == io::ErrorKind::Interrupted`. + pub fn save(&mut self) -> io::Result { + try!(fs::create_dir_all(self.save_dir)); + let path = self.gen_safe_file_path(self.save_dir); + self.save_as(&path).map(move |_| path) } + /// Get the filename of this entry, if supplied. + /// + /// ##Warning + /// You should treat this value as untrustworthy because it is an arbitrary string provided by + /// the client. You should *not* blindly append it to a directory path and save the file there, + /// as such behavior could easily be exploited by a malicious client. pub fn filename(&self) -> Option<&str> { self.filename.as_ref().map(String::as_ref) } - /// Get the content type of this file. - /// On the client, it is guessed by the file extension. - /// On the server, it is retrieved from the request or assumed to be - /// `application/octet-stream`. + /// Get the MIME type (`Content-Type` value) of this file, if supplied by the client, + /// or `"applicaton/octet-stream"` otherwise. pub fn content_type(&self) -> Mime { self.content_type.clone() } + + /// The save directory assigned to this file field by the `Multipart` instance it was read + /// from. + pub fn save_dir(&self) -> &Path { + self.save_dir + } + + fn gen_safe_file_path(&self, dir: &Path) -> PathBuf { + self.filename().map(Path::new) + .and_then(Path::file_name) //Make sure there's no path separators in the filename + .map_or_else( + || dir.join(::random_alphanumeric(8)), + |filename| dir.join(filename), + ) + } } impl<'a, R: Read> Read for MultipartFile<'a, R> { fn read(&mut self, buf: &mut [u8]) -> io::Result{ - self.reader.read(buf) + self.stream.read(buf) + } +} + +impl<'a, R: Read> BufRead for MultipartFile<'a, R> { + fn fill_buf(&mut self) -> io::Result<&[u8]> { + self.stream.fill_buf() + } + + fn consume(&mut self, amt: usize) { + self.stream.consume(amt) } } /// A result of `Multipart::save_all()`. pub struct Entries { - /// The text files of the multipart request + /// The text fields of the multipart request. pub fields: HashMap, - /// A list of file field names and their save results. + /// A map of file field names to their save results. pub files: HashMap>, /// The directory the files were saved under. pub dir: PathBuf, @@ -432,3 +469,14 @@ impl Entries { } } +fn retry_on_interrupt(mut do_fn: F) -> io::Result where F: FnMut() -> io::Result { + loop { + match do_fn() { + Ok(val) => return Ok(val), + Err(err) => if err.kind() != io::ErrorKind::Interrupted { + return Err(err); + }, + } + } +} + diff --git a/multipart/tests/local.rs b/multipart/tests/local.rs index 70b3cf42c..29db03eb0 100644 --- a/multipart/tests/local.rs +++ b/multipart/tests/local.rs @@ -47,16 +47,17 @@ fn test_server(buf: HttpBuffer) { #[derive(Default, Debug)] struct MockClientRequest { boundary: Option, - content_len: Option, + content_len: Option, } impl ClientRequest for MockClientRequest { type Stream = HttpBuffer; type Error = io::Error; - fn apply_headers(&mut self, boundary: &str, content_len: Option) { + fn apply_headers(&mut self, boundary: &str, content_len: Option) -> bool { self.boundary = Some(boundary.into()); self.content_len = content_len; + true } fn open_stream(self) -> Result { @@ -71,7 +72,7 @@ impl ClientRequest for MockClientRequest { struct HttpBuffer { buf: Vec, boundary: String, - content_len: Option, + content_len: Option, } impl HttpBuffer { @@ -112,7 +113,7 @@ impl ClientStream for HttpBuffer { struct ServerBuffer<'a> { data: &'a [u8], boundary: &'a str, - content_len: Option, + content_len: Option, } impl<'a> Read for ServerBuffer<'a> { From 2221aa0e9f667a61116984b1044ff1864855fb69 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Tue, 14 Jul 2015 02:37:29 -0700 Subject: [PATCH 065/453] Documentation nigglings --- multipart/src/client/mod.rs | 25 +++++++------------------ multipart/src/client/sized.rs | 17 ++++++++++++++--- multipart/src/lib.rs | 7 ++++--- multipart/src/server/mod.rs | 2 +- 4 files changed, 26 insertions(+), 25 deletions(-) diff --git a/multipart/src/client/mod.rs b/multipart/src/client/mod.rs index fd9e5194e..74a9b3609 100644 --- a/multipart/src/client/mod.rs +++ b/multipart/src/client/mod.rs @@ -17,19 +17,6 @@ pub mod hyper; mod sized; -/// A wrapper around a request object that measures the request body and sets the `Content-Length` -/// header to its size in bytes. -/// -/// Sized requests are more human-readable and use less bandwidth -/// (as chunking adds [visual noise and overhead][chunked-example]), -/// but they must be able to load their entirety, including the contents of all files -/// and streams, into memory so the request body can be measured. -/// -/// You should really only use sized requests if you intend to inspect the data manually on the -/// server side, as it will produce a more human-readable request body. Also, of course, if the -/// server doesn't support chunked requests or otherwise rejects them. -/// -/// [chunked-example]: http://en.wikipedia.org/wiki/Chunked_transfer_encoding#Example pub use self::sized::SizedRequest; @@ -38,9 +25,6 @@ pub use self::sized::SizedRequest; /// Though they perform I/O, the `.write_*()` methods do not return `io::Result<_>` in order to /// facilitate method chaining. Upon the first error, all subsequent API calls will be no-ops until /// `.send()` is called, at which point the error will be reported. -/// -/// If you don't want to consume data handles (`File`, etc.), `write_file()` and `write_stream()` -/// also accept `&mut` versions. pub struct Multipart { stream: S, boundary: String, @@ -103,6 +87,8 @@ impl Multipart { /// If you want to set these values manually, or use another type that implements `Read`, /// use `.write_stream()`. /// + /// `name` can be either `String` or `&str`, and `path` can be `PathBuf` or `&Path`. + /// /// ##Errors /// If there was a problem opening the file (was a directory or didn't exist), /// or if something went wrong with the HTTP stream. @@ -127,15 +113,18 @@ impl Multipart { /// Write a byte stream to the multipart request as a file field, supplying `filename` if given, /// and `content_type` if given or `"application/octet-stream"` if not. /// + /// `name` can be either `String` or `&str`, and `read` can take the `Read` by-value or + /// with an `&mut` borrow. + /// /// ##Warning /// The given `Read` **must** be able to read to EOF (end of file/no more data), meaning /// `Read::read()` returns `Ok(0)`. If it never returns EOF it will be read to infinity /// and the request will never be completed. /// - /// When using `sized::SizedRequest` this also can cause out-of-control memory usage as the + /// When using `SizedRequest` this also can cause out-of-control memory usage as the /// multipart data has to be written to an in-memory buffer so its size can be calculated. /// - /// Use `Read::take(usize)` if you wish to send data from a `Read` + /// Use `Read::take()` if you wish to send data from a `Read` /// that will never return EOF otherwise. /// /// ##Errors diff --git a/multipart/src/client/sized.rs b/multipart/src/client/sized.rs index 019a9b15a..a5dd18145 100644 --- a/multipart/src/client/sized.rs +++ b/multipart/src/client/sized.rs @@ -5,8 +5,19 @@ use client::{HttpRequest, HttpStream}; use std::io; use std::io::prelude::*; -/// A wrapper around `HttpRequest` that writes the multipart data to an in-memory buffer so its -/// size can be calculated and set in the request as the `Content-Length` header. +/// A wrapper around a request object that measures the request body and sets the `Content-Length` +/// header to its size in bytes. +/// +/// Sized requests are more human-readable and use less bandwidth +/// (as chunking adds [visual noise and overhead][chunked-example]), +/// but they must be able to load their entirety, including the contents of all files +/// and streams, into memory so the request body can be measured. +/// +/// You should really only use sized requests if you intend to inspect the data manually on the +/// server side, as it will produce a more human-readable request body. Also, of course, if the +/// server doesn't support chunked requests or otherwise rejects them. +/// +/// [chunked-example]: http://en.wikipedia.org/wiki/Chunked_transfer_encoding#Example pub struct SizedRequest { inner: R, buffer: Vec, @@ -52,7 +63,7 @@ where ::Error: From { impl HttpStream for SizedRequest where ::Error: From { - type Request = R; + type Request = Self; type Response = <::Stream as HttpStream>::Response; type Error = <::Stream as HttpStream>::Error; diff --git a/multipart/src/lib.rs b/multipart/src/lib.rs index cb65c61a7..d15004dbb 100644 --- a/multipart/src/lib.rs +++ b/multipart/src/lib.rs @@ -1,8 +1,9 @@ //! Client- and server-side abstractions for HTTP `multipart/form-data` requests. //! -//! Features: -//! * `hyper`: Enable client- and server-side integration with -//! [Hyper](https:://github.com/hyperium/hyper) +//! Features: +//! +//! * `hyper`: Enable client- and server-side integration with the +//! [Hyper](https:://github.com/hyperium/hyper) HTTP library. #![warn(missing_docs)] #[macro_use] extern crate log; extern crate env_logger; diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index 7ad7c7c28..a4cee1587 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -1,4 +1,4 @@ -//! The server-side abstraction for multipart requests requests. +//! The server-side abstraction for multipart requests. //! //! Use this when you are implementing an HTTP server and want to //! to accept, parse, and serve HTTP `multipart/form-data` requests (file uploads). From b176b9801e4797bea28fccc350a71c9fe27158bb Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Tue, 14 Jul 2015 02:58:41 -0700 Subject: [PATCH 066/453] Bump version --- multipart/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 78dec5d6e..397de69dc 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "multipart" -version = "0.1.1" +version = "0.2.0" authors = ["Austin Bonander "] description = "An extension to the Hyper HTTP library that provides support for POST multipart/form-data requests on for both client and server." From e6410db50330b1891da386e6ed4e76af82478677 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Tue, 14 Jul 2015 15:00:42 -0700 Subject: [PATCH 067/453] Update README.md --- multipart/README.md | 77 ++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 70 insertions(+), 7 deletions(-) diff --git a/multipart/README.md b/multipart/README.md index 3dd316448..23a784842 100644 --- a/multipart/README.md +++ b/multipart/README.md @@ -1,19 +1,82 @@ Multipart + Hyper [![Build Status](https://travis-ci.org/cybergeek94/multipart.svg?branch=master)](https://travis-ci.org/cybergeek94/multipart) ========= -An extension to [Hyper][1] that adds support for HTTP multipart (`Content-Type: multipart/form-data`) requests. +Client- and server-side abstractions for HTTP file uploads (POST requests with `Content-Type: multipart/form-data`). -See `src/bin/multipart_server.rs` for server-side example (used in testing). +Provides integration with [Hyper](https://github.com/hyperium/hyper) via the `hyper` feature. More to come! + +Usage +----- + +In your `Cargo.toml`: +```toml +# Currently only useful with `hyper` and `url` crates: +[dependencies] +hyper = "*" +url = "*" + +[dependencies.multipart] +version = "0.2" +# The `hyper` feature enables trait impls for Hyper's request objects. +# RFC: should this be set by default? +features = ["hyper"] +``` + +Client-side example (using Hyper): +```rust +extern crate hyper; +extern crate multipart; +extern crate url; + +use hyper::client::request::Request; +use hyper::method::Method; + +use multipart::client::Multipart; + +use url::Url; + +fn main() { + let url = Url::parse("127.0.0.1").unwrap(); + let request = Request::new(Method::Post, url).unwrap(); + + let mut response = Multipart::from_request(request).unwrap() + .write_text("hello", "world") + .write_file("my_file", "my_text_data.txt") + .send().unwrap(); + + // Read response... +} +``` + +Server-side example (using Hyper): +```rust +use hyper::net::Fresh; +use hyper::server::{Server, Request, Response}; + +use multipart::server::Multipart; +use multipart::server::hyper::Switch; + +fn handle_regular<'a, 'k>(req: Request<'a, 'k>, res: Response<'a, Fresh>) { + // handle things here +} + +fn handle_multipart<'a, 'k>(mut multi: Multipart>, res: Response<'a, Fresh>) { + multi.foreach_entry(|entry| println!("Multipart entry: {:?}", entry)).unwrap(); +} + +fn main() { + Server::http("0.0.0.0:0").unwrap() + .handle(Switch::new(handle_regular, handle_multipart)) + .unwrap(); +} +``` ####[Documentation][2] -#####TODO: -- [x] Remove excess debug statements +#####TODO: - [ ] Fill out README and provide examples -- [ ] Improve documentation - [ ] Add support for multiple files per field (nested boundaries) -- [ ] Expand test coverage -- [ ] Integrate with future HTTP compression extension(s) + [1]: https://github.com/hyperium/hyper [2]: http://rust-ci.org/cybergeek94/multipart/doc/multipart/ From 4a90185108b9dfedd19dadc319470135197ed6b9 Mon Sep 17 00:00:00 2001 From: Nicolas Cherel Date: Thu, 16 Jul 2015 14:03:29 +0200 Subject: [PATCH 068/453] fixes boundary param match for proper multipart request detection --- multipart/src/server/hyper.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/src/server/hyper.rs b/multipart/src/server/hyper.rs index 086714c76..e049b9f70 100644 --- a/multipart/src/server/hyper.rs +++ b/multipart/src/server/hyper.rs @@ -83,7 +83,7 @@ impl<'a, 'b> HttpRequest for Request<'a, 'b> { params.iter().find(|&&(ref name, _)| match *name { - Attr::Ext(ref name) => "boundary" == name, + Attr::Boundary => true, _ => false, } ).and_then(|&(_, ref val)| From 1958f371a0bbfadfd8a59117adf3b919941c7e4a Mon Sep 17 00:00:00 2001 From: Nicolas Cherel Date: Thu, 16 Jul 2015 16:54:11 +0200 Subject: [PATCH 069/453] =?UTF-8?q?fixes=20a=20panic!=20:=20when=20searchi?= =?UTF-8?q?ng=20from=20=E2=80=98val=5Fstart=5Fidx=E2=80=99=20in=20?= =?UTF-8?q?=E2=80=98haystack=E2=80=99,=20result=20has=20a=20=E2=80=98val?= =?UTF-8?q?=5Fstart=5Fidx=E2=80=99=20offset?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- multipart/src/server/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index a4cee1587..d733eb635 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -232,7 +232,7 @@ impl ContentDisp { /// Get the string after `needle` in `haystack`, stopping before `end_val_delim` fn get_str_after<'a>(needle: &str, end_val_delim: char, haystack: &'a str) -> Option<(&'a str, &'a str)> { let val_start_idx = try_opt!(haystack.find(needle)) + needle.len(); - let val_end_idx = try_opt!(haystack[val_start_idx..].find(end_val_delim)); + let val_end_idx = try_opt!(haystack[val_start_idx..].find(end_val_delim)) + val_start_idx; Some((&haystack[val_start_idx..val_end_idx], &haystack[val_end_idx..])) } From ebc7d18f742ee10b7f267954a9547588035435ed Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Thu, 16 Jul 2015 16:56:31 -0700 Subject: [PATCH 070/453] Add feature flags for client, server --- multipart/Cargo.toml | 5 +++++ multipart/src/lib.rs | 2 ++ 2 files changed, 7 insertions(+) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 397de69dc..f3516f9c1 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -14,6 +14,11 @@ license = "MIT" keywords = ["form-data", "hyper", "http", "post", "upload"] +[features] +server = [] +client = [] +default = ["hyper", "server", "client"] + [dependencies] log = "*" env_logger = "*" diff --git a/multipart/src/lib.rs b/multipart/src/lib.rs index d15004dbb..3c9eb48c6 100644 --- a/multipart/src/lib.rs +++ b/multipart/src/lib.rs @@ -31,7 +31,9 @@ macro_rules! chain_result { ) } +#[cfg(feature = "client")] pub mod client; +#[cfg(feature = "server")] pub mod server; const BOUNDARY_LEN: usize = 16; From 9371d6b5a35f1ca2b307d118ec6447c3431429e9 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Thu, 16 Jul 2015 17:00:57 -0700 Subject: [PATCH 071/453] Bump version number --- multipart/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index f3516f9c1..dde35b5a1 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "multipart" -version = "0.2.0" +version = "0.2.1" authors = ["Austin Bonander "] description = "An extension to the Hyper HTTP library that provides support for POST multipart/form-data requests on for both client and server." From 1c65f413ec4cf3049d7fd73210e190d6aa7a52ce Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Thu, 16 Jul 2015 17:07:17 -0700 Subject: [PATCH 072/453] Update README.md --- multipart/README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/multipart/README.md b/multipart/README.md index 23a784842..03ac2feee 100644 --- a/multipart/README.md +++ b/multipart/README.md @@ -16,13 +16,13 @@ hyper = "*" url = "*" [dependencies.multipart] -version = "0.2" -# The `hyper` feature enables trait impls for Hyper's request objects. -# RFC: should this be set by default? -features = ["hyper"] +version = "0.2.1" +# You can also select which features to compile: +# default-features = false +# features = ["hyper", "server", "client"] ``` -Client-side example (using Hyper): +Client-side example using Hyper (`--features hyper,client` or default): ```rust extern crate hyper; extern crate multipart; @@ -48,7 +48,7 @@ fn main() { } ``` -Server-side example (using Hyper): +Server-side example using Hyper (`--features hyper,server` or default`): ```rust use hyper::net::Fresh; use hyper::server::{Server, Request, Response}; @@ -74,8 +74,8 @@ fn main() { ####[Documentation][2] #####TODO: -- [ ] Fill out README and provide examples - [ ] Add support for multiple files per field (nested boundaries) +- [ ] Expand integration tests to cover more cases [1]: https://github.com/hyperium/hyper From 197b5d2fbeca773ca1735a01efbc84acc938284f Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Thu, 16 Jul 2015 17:13:53 -0700 Subject: [PATCH 073/453] Adjust documentation --- multipart/src/client/hyper.rs | 2 +- multipart/src/client/mod.rs | 3 ++- multipart/src/lib.rs | 10 ++++++++-- multipart/src/server/hyper.rs | 2 +- multipart/src/server/mod.rs | 3 ++- 5 files changed, 14 insertions(+), 6 deletions(-) diff --git a/multipart/src/client/hyper.rs b/multipart/src/client/hyper.rs index 0e4dff889..f4ef10017 100644 --- a/multipart/src/client/hyper.rs +++ b/multipart/src/client/hyper.rs @@ -1,5 +1,5 @@ //! Client-side integration with [Hyper](https://github.com/hyperium/hyper). -//! Enabled with the `hyper` feature. +//! Enabled with the `hyper` feature (on by default). //! //! Contains `impl HttpRequest for Request` and `impl HttpStream for Request`. use hyper::client::request::Request; diff --git a/multipart/src/client/mod.rs b/multipart/src/client/mod.rs index 74a9b3609..c406b1fd1 100644 --- a/multipart/src/client/mod.rs +++ b/multipart/src/client/mod.rs @@ -1,4 +1,5 @@ -//! The client-side abstraction for multipart requests. +//! The client-side abstraction for multipart requests. Enabled with the `client` feature (on by +//! default). //! //! Use this when sending POST requests with files to a server. use mime::Mime; diff --git a/multipart/src/lib.rs b/multipart/src/lib.rs index 3c9eb48c6..901e5e8c8 100644 --- a/multipart/src/lib.rs +++ b/multipart/src/lib.rs @@ -1,9 +1,15 @@ //! Client- and server-side abstractions for HTTP `multipart/form-data` requests. //! //! Features: +//! +//! * `client` (default): Enable the client-side abstractions for multipart requests. If the +//! `hyper` feature is also set, enables integration with the Hyper HTTP client API. //! -//! * `hyper`: Enable client- and server-side integration with the -//! [Hyper](https:://github.com/hyperium/hyper) HTTP library. +//! * `server` (default): Enable the server-side abstractions for multipart requests. If the +//! `hyper` feature is also set, enables integration with the Hyper HTTP server API. +//! +//! * `hyper` (default): Enable integration with the [Hyper](https:://github.com/hyperium/hyper) HTTP library +//! for client and/or server depending on which other feature flags are set. #![warn(missing_docs)] #[macro_use] extern crate log; extern crate env_logger; diff --git a/multipart/src/server/hyper.rs b/multipart/src/server/hyper.rs index 086714c76..87d2d38c5 100644 --- a/multipart/src/server/hyper.rs +++ b/multipart/src/server/hyper.rs @@ -1,5 +1,5 @@ //! Server-side integration with [Hyper](https://github.com/hyperium/hyper). -//! Enabled with the `hyper` feature. +//! Enabled with the `hyper` feature (on by default). use hyper::net::Fresh; use hyper::header::ContentType; use hyper::method::Method; diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index a4cee1587..0fa5f6431 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -1,4 +1,5 @@ -//! The server-side abstraction for multipart requests. +//! The server-side abstraction for multipart requests. Enabled with the `server` feature (on by +//! default). //! //! Use this when you are implementing an HTTP server and want to //! to accept, parse, and serve HTTP `multipart/form-data` requests (file uploads). From afc7c322abaa42628c45e59c838e124937896503 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Fri, 17 Jul 2015 03:30:00 -0700 Subject: [PATCH 074/453] Rewrite test to be more robust, fix testing on Windows --- multipart/Cargo.toml | 4 ++ multipart/src/client/mod.rs | 21 +++---- multipart/tests/local.rs | 120 ++++++++++++++++++++++++++++++------ 3 files changed, 115 insertions(+), 30 deletions(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index dde35b5a1..8d37ed48f 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -30,3 +30,7 @@ rand = "*" version = "*" optional = true +[dev-dependencies.hyper] +version = "*" +# Disable SSL for testing purposes +default-features = false diff --git a/multipart/src/client/mod.rs b/multipart/src/client/mod.rs index c406b1fd1..7fadc346b 100644 --- a/multipart/src/client/mod.rs +++ b/multipart/src/client/mod.rs @@ -4,9 +4,6 @@ //! Use this when sending POST requests with files to a server. use mime::Mime; -use std::borrow::{Borrow, BorrowMut}; -use std::convert::AsRef; - use std::fs::File; use std::io; use std::io::prelude::*; @@ -70,11 +67,11 @@ impl Multipart { /// /// ##Errors /// If something went wrong with the HTTP stream. - pub fn write_text, V: Borrow>(mut self, name: N, val: V) -> Self { + pub fn write_text, V: AsRef>(mut self, name: N, val: V) -> Self { if self.last_err.is_none() { self.last_err = chain_result! { - self.write_field_headers(name.borrow(), None, None), - self.write_line(val.borrow()), + self.write_field_headers(name.as_ref(), None, None), + self.write_line(val.as_ref()), self.write_boundary() }.err().map(|err| err.into()) } @@ -93,7 +90,7 @@ impl Multipart { /// ##Errors /// If there was a problem opening the file (was a directory or didn't exist), /// or if something went wrong with the HTTP stream. - pub fn write_file, P: AsRef>(mut self, name: N, path: P) -> Self { + pub fn write_file, P: AsRef>(mut self, name: N, path: P) -> Self { if self.last_err.is_none() { let path = path.as_ref(); @@ -101,7 +98,7 @@ impl Multipart { { // New borrow scope so we can reborrow `file` after let content_type = ::mime_guess::guess_mime_type(path); let filename = path.file_name().and_then(|filename| filename.to_str()); - self.write_field_headers(name.borrow(), filename, Some(content_type)) + self.write_field_headers(name.as_ref(), filename, Some(content_type)) }, File::open(path).and_then(|ref mut file| io::copy(file, &mut self.stream)), self.write_boundary() @@ -131,15 +128,15 @@ impl Multipart { /// ##Errors /// If the reader returned an error, or if something went wrong with the HTTP stream. // RFC: How to format this declaration? - pub fn write_stream, St: Read, St_: BorrowMut>( - mut self, name: N, mut read: St_, filename: Option<&str>, content_type: Option + pub fn write_stream, St: Read>( + mut self, name: N, read: &mut St, filename: Option<&str>, content_type: Option ) -> Self { if self.last_err.is_none() { let content_type = content_type.unwrap_or_else(::mime_guess::octet_stream); self.last_err = chain_result! { - self.write_field_headers(name.borrow(), filename, Some(content_type)), - io::copy(read.borrow_mut(), &mut self.stream), + self.write_field_headers(name.as_ref(), filename, Some(content_type)), + io::copy(read, &mut self.stream), self.write_boundary() }.err().map(|err| err.into()); } diff --git a/multipart/tests/local.rs b/multipart/tests/local.rs index 29db03eb0..05a62affb 100644 --- a/multipart/tests/local.rs +++ b/multipart/tests/local.rs @@ -1,6 +1,7 @@ #[macro_use] extern crate log; extern crate env_logger; +extern crate rand; extern crate multipart; @@ -9,47 +10,130 @@ use multipart::client::HttpStream as ClientStream; use multipart::server::HttpRequest as ServerRequest; +use rand::Rng; +use rand::distributions::{Range, Sample}; + +use std::collections::HashMap; use std::io; use std::io::prelude::*; +struct TestFields { + texts: HashMap, + files: HashMap>, +} + #[test] fn local_test() { - let buf = test_client(); - test_server(buf); + let test_fields = gen_test_fields(); + + let buf = test_client(&test_fields); + test_server(buf, test_fields); +} + +fn gen_test_fields() -> TestFields { + const MIN_FIELDS: usize = 1; + const MAX_FIELDS: usize = 5; + + let texts_count = gen_range(MIN_FIELDS, MAX_FIELDS); + let files_count = gen_range(MIN_FIELDS, MAX_FIELDS); + + TestFields { + texts: (0..texts_count).map(|_| (gen_string(), gen_string())).collect(), + files: (0..files_count).map(|_| (gen_string(), gen_bytes())).collect(), + } +} + +fn gen_range(min: usize, max: usize) -> usize { + Range::new(min, max).sample(&mut rand::weak_rng()) +} + +fn gen_string() -> String { + const MIN_LEN: usize = 3; + const MAX_LEN: usize = 12; + + let mut rng = rand::weak_rng(); + let str_len = gen_range(MIN_LEN, MAX_LEN); + + rng.gen_ascii_chars().take(str_len).collect() } -fn test_client() -> HttpBuffer { +fn gen_bytes() -> Vec { + const MIN_LEN: usize = 64; + const MAX_LEN: usize = 1024; + + let mut rng = rand::weak_rng(); + let bytes_len = gen_range(MIN_LEN, MAX_LEN); + + let mut vec = vec![0u8; bytes_len]; + rng.fill_bytes(&mut vec); + vec +} + + +fn test_client(test_fields: &TestFields) -> HttpBuffer { use multipart::client::Multipart; let request = MockClientRequest::default(); + + let mut test_files = test_fields.files.iter(); + + let mut multipart = Multipart::from_request(request).unwrap(); - Multipart::from_request(request).unwrap() - .write_text("hello", "world") - .write_text("goodnight", "sun") - .send().unwrap() + for (name, text) in &test_fields.texts { + if let Some((file_name, file)) = test_files.next() { + multipart = multipart.write_stream(file_name, &mut &**file, None, None); + } + + multipart = multipart.write_text(name, text); + } + + multipart.send().unwrap() } -fn test_server(buf: HttpBuffer) { - use multipart::server::Multipart; +fn test_server(buf: HttpBuffer, mut fields: TestFields) { + use multipart::server::{Multipart, MultipartData}; let mut multipart = Multipart::from_request(buf.for_server()) .unwrap_or_else(|_| panic!("Buffer should be multipart!")); - while let Ok(Some(field)) = multipart.read_entry() { - match &*field.name { - "hello" => assert_eq!(field.data.as_text(), Some("world")), - "goodnight" => assert_eq!(field.data.as_text(), Some("sun")), - _ => panic!("Unexpected field: {:?}", field), + while let Ok(Some(mut field)) = multipart.read_entry() { + match field.data { + MultipartData::Text(text) => { + let test_text = fields.texts.remove(&field.name).unwrap(); + assert!( + text == test_text, + "Expected {:?} for {:?} got {:?}", + text, field.name, test_text + ); + + }, + MultipartData::File(ref mut file) => { + let test_bytes = fields.files.remove(&field.name).unwrap(); + + let mut bytes = Vec::with_capacity(test_bytes.len()); + file.read_to_end(&mut bytes).unwrap(); + + assert!(bytes == test_bytes, "Unexpected data for {:?}", field.name); + }, } } + + assert!(fields.texts.is_empty(), "Text fields were not exhausted! Text fields: {:?}", fields.texts); + assert!(fields.files.is_empty(), "File fields were not exhausted!"); } #[derive(Default, Debug)] -struct MockClientRequest { +pub struct MockClientRequest { boundary: Option, content_len: Option, } +impl MockClientRequest { + pub fn new() -> MockClientRequest { + Self::default() + } +} + impl ClientRequest for MockClientRequest { type Stream = HttpBuffer; type Error = io::Error; @@ -69,14 +153,14 @@ impl ClientRequest for MockClientRequest { } #[derive(Debug)] -struct HttpBuffer { +pub struct HttpBuffer { buf: Vec, boundary: String, content_len: Option, } impl HttpBuffer { - fn for_server(&self) -> ServerBuffer { + pub fn for_server(&self) -> ServerBuffer { ServerBuffer { data: &self.buf, boundary: &self.boundary, @@ -110,7 +194,7 @@ impl ClientStream for HttpBuffer { } #[derive(Debug)] -struct ServerBuffer<'a> { +pub struct ServerBuffer<'a> { data: &'a [u8], boundary: &'a str, content_len: Option, From d45916a0ea46be94663ce0f59cf1663d77093475 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Fri, 17 Jul 2015 04:24:17 -0700 Subject: [PATCH 075/453] WIP logic fixes and cleanup --- multipart/src/client/mod.rs | 21 ++++++++++++--------- multipart/src/lib.rs | 17 ++++------------- multipart/src/server/boundary.rs | 6 ++++-- multipart/src/server/mod.rs | 18 +++++++++++++++--- 4 files changed, 35 insertions(+), 27 deletions(-) diff --git a/multipart/src/client/mod.rs b/multipart/src/client/mod.rs index 7fadc346b..b829048b2 100644 --- a/multipart/src/client/mod.rs +++ b/multipart/src/client/mod.rs @@ -17,6 +17,8 @@ mod sized; pub use self::sized::SizedRequest; +const BOUNDARY_LEN: usize = 16; + /// The entry point of the client-side multipart API. /// @@ -36,7 +38,7 @@ impl Multipart { /// ## Returns Error /// If `req.open_stream()` returns an error. pub fn from_request(mut req: R) -> Result, R::Error> { - let boundary = ::gen_boundary(); + let boundary = ::random_alphanumeric(BOUNDARY_LEN); req.apply_headers(&boundary, None); let stream = try!(req.open_stream()); @@ -58,6 +60,9 @@ impl Multipart { /// Remove and return the last error to occur, allowing subsequent API calls to proceed /// normally. + /// + /// ##Warning + /// If an error occurred during a write, the request body may be corrupt. pub fn take_err(&mut self) -> Option { self.last_err.take() } @@ -71,8 +76,7 @@ impl Multipart { if self.last_err.is_none() { self.last_err = chain_result! { self.write_field_headers(name.as_ref(), None, None), - self.write_line(val.as_ref()), - self.write_boundary() + self.write_line(val.as_ref()) }.err().map(|err| err.into()) } @@ -100,8 +104,7 @@ impl Multipart { let filename = path.file_name().and_then(|filename| filename.to_str()); self.write_field_headers(name.as_ref(), filename, Some(content_type)) }, - File::open(path).and_then(|ref mut file| io::copy(file, &mut self.stream)), - self.write_boundary() + File::open(path).and_then(|ref mut file| io::copy(file, &mut self.stream)) }.err().map(|err| err.into()); } @@ -137,7 +140,6 @@ impl Multipart { self.last_err = chain_result! { self.write_field_headers(name.as_ref(), filename, Some(content_type)), io::copy(read, &mut self.stream), - self.write_boundary() }.err().map(|err| err.into()); } @@ -147,6 +149,8 @@ impl Multipart { fn write_field_headers(&mut self, name: &str, filename: Option<&str>, content_type: Option) -> io::Result<()> { chain_result! { + // Write the end boundary to the previous field first + if self.data_written { self.write_boundary() } else { self.data_written = true; Ok(()) }, write!(self.stream, "Content-Disposition: form-data; name=\"{}\"", name), filename.map(|filename| write!(self.stream, "; filename=\"{}\"", filename)) .unwrap_or(Ok(())), @@ -161,8 +165,7 @@ impl Multipart { } fn write_boundary(&mut self) -> io::Result<()> { - write!(self.stream, "{}\r\n", self.boundary) - .map(|res| { self.data_written = true; res }) + write!(self.stream, "--{}\r\n", self.boundary) } /// Finalize the request and return the response from the server, or the last error if set. @@ -171,7 +174,7 @@ impl Multipart { None => { if self.data_written { // Write two hyphens after the last boundary occurrence. - try!(self.stream.write(b"--")); + try!(write!(self.stream, "--{}--", self.boundary)); } self.stream.finish() diff --git a/multipart/src/lib.rs b/multipart/src/lib.rs index 901e5e8c8..447d91d4a 100644 --- a/multipart/src/lib.rs +++ b/multipart/src/lib.rs @@ -25,16 +25,13 @@ use rand::Rng; use std::path::PathBuf; -macro_rules! try_all { - ($first_expr:expr, $($try_expr:expr),*) => ( - try!($first_expr $(.and_then(|_| $try_expr))*); - ) -} - macro_rules! chain_result { ($first_expr:expr, $($try_expr:expr),*) => ( $first_expr $(.and_then(|_| $try_expr))* - ) + ); + ($first_expr:expr, $($try_expr:expr),*,) => ( + chain_result! { $first_expr, $($try_expr),* } + ); } #[cfg(feature = "client")] @@ -42,7 +39,6 @@ pub mod client; #[cfg(feature = "server")] pub mod server; -const BOUNDARY_LEN: usize = 16; const DIRNAME_LEN: usize = 12; fn temp_dir() -> PathBuf { @@ -53,8 +49,3 @@ fn random_alphanumeric(len: usize) -> String { rand::thread_rng().gen_ascii_chars().take(len).collect() } -fn gen_boundary() -> String { - let mut boundary = "--".to_owned(); - boundary.extend(rand::thread_rng().gen_ascii_chars().take(BOUNDARY_LEN)); - boundary -} diff --git a/multipart/src/server/boundary.rs b/multipart/src/server/boundary.rs index d146e1959..b1018bc07 100644 --- a/multipart/src/server/boundary.rs +++ b/multipart/src/server/boundary.rs @@ -31,12 +31,13 @@ impl BoundaryReader where R: Read { let buf = try!(self.buffer.fill_buf()); if !self.boundary_read { + let last_search_idx = self.search_idx; let lookahead_iter = buf[self.search_idx..].windows(self.boundary.len()).enumerate(); for (search_idx, maybe_boundary) in lookahead_iter { if maybe_boundary[0] == self.boundary[0] { self.boundary_read = self.boundary == maybe_boundary; - self.search_idx = search_idx; + self.search_idx = last_search_idx + search_idx; if self.boundary_read { break; @@ -64,7 +65,8 @@ impl BoundaryReader where R: Read { self.buffer.consume(consume_amt); self.search_idx = 0; self.boundary_read = false; -Ok(()) + + Ok(()) } // Keeping this around to support nested boundaries later. diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index 3a0340e72..a0ccc8cc5 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -41,6 +41,7 @@ macro_rules! try_opt ( pub struct Multipart { source: BoundaryReader, line_buf: String, + at_end: bool, /// The directory for saving files in this request. /// By default, this is set to a subdirectory of `std::env::temp_dir()` with a /// random alphanumeric name. @@ -65,6 +66,7 @@ impl Multipart where R: HttpRequest { Multipart { source: BoundaryReader::from_reader(req, boundary), line_buf: String::new(), + at_end: false, save_dir: ::temp_dir(), } ) @@ -77,7 +79,19 @@ impl Multipart where R: HttpRequest { /// If the previously returned entry had contents of type `MultipartField::File`, /// calling this again will discard any unread contents of that entry. pub fn read_entry(&mut self) -> io::Result>> { - MultipartField::read_from(self) + if self.at_end { return Ok(None); } + + let mut byte2 = [0u8; 2]; + try!(self.source.consume_boundary()); + try!(self.source.read(&mut byte2)); + + self.at_end = &byte2 == b"--"; + + if !self.at_end { + MultipartField::read_from(self) + } else { + Ok(None) + } } fn read_content_disposition(&mut self) -> io::Result> { @@ -262,8 +276,6 @@ pub struct MultipartField<'a, R: 'a> { impl<'a, R: HttpRequest + 'a> MultipartField<'a, R> { fn read_from(multipart: &'a mut Multipart) -> io::Result>> { - try!(multipart.source.consume_boundary()); - let cont_disp = match multipart.read_content_disposition() { Ok(Some(cont_disp)) => cont_disp, Ok(None) => return Ok(None), From a9e08ae3826212266911e8df2e7a3692e2f16da9 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Fri, 17 Jul 2015 05:16:28 -0700 Subject: [PATCH 076/453] Fix everything! --- multipart/src/client/mod.rs | 22 ++++++++-------------- multipart/src/lib.rs | 3 --- multipart/src/server/mod.rs | 4 ++-- multipart/tests/local.rs | 25 ++++++++++++++++++++----- 4 files changed, 30 insertions(+), 24 deletions(-) diff --git a/multipart/src/client/mod.rs b/multipart/src/client/mod.rs index b829048b2..e10c50841 100644 --- a/multipart/src/client/mod.rs +++ b/multipart/src/client/mod.rs @@ -76,7 +76,7 @@ impl Multipart { if self.last_err.is_none() { self.last_err = chain_result! { self.write_field_headers(name.as_ref(), None, None), - self.write_line(val.as_ref()) + self.stream.write_all(val.as_ref().as_bytes()) }.err().map(|err| err.into()) } @@ -139,7 +139,7 @@ impl Multipart { self.last_err = chain_result! { self.write_field_headers(name.as_ref(), filename, Some(content_type)), - io::copy(read, &mut self.stream), + io::copy(read, &mut self.stream) }.err().map(|err| err.into()); } @@ -148,33 +148,27 @@ impl Multipart { fn write_field_headers(&mut self, name: &str, filename: Option<&str>, content_type: Option) -> io::Result<()> { + self.data_written = true; + chain_result! { - // Write the end boundary to the previous field first - if self.data_written { self.write_boundary() } else { self.data_written = true; Ok(()) }, + // Write the first boundary, or the boundary for the previous field. + write!(self.stream, "\r\n--{}\r\n", self.boundary), write!(self.stream, "Content-Disposition: form-data; name=\"{}\"", name), filename.map(|filename| write!(self.stream, "; filename=\"{}\"", filename)) .unwrap_or(Ok(())), content_type.map(|content_type| write!(self.stream, "\r\nContent-Type: {}", content_type)) .unwrap_or(Ok(())), - self.write_line("\r\n") + self.stream.write_all(b"\r\n\r\n") } } - fn write_line(&mut self, line: &str) -> io::Result<()> { - write!(self.stream, "{}\r\n", line) - } - - fn write_boundary(&mut self) -> io::Result<()> { - write!(self.stream, "--{}\r\n", self.boundary) - } - /// Finalize the request and return the response from the server, or the last error if set. pub fn send(mut self) -> Result { match self.last_err { None => { if self.data_written { // Write two hyphens after the last boundary occurrence. - try!(write!(self.stream, "--{}--", self.boundary)); + try!(write!(self.stream, "\r\n--{}--", self.boundary)); } self.stream.finish() diff --git a/multipart/src/lib.rs b/multipart/src/lib.rs index 447d91d4a..8b29176df 100644 --- a/multipart/src/lib.rs +++ b/multipart/src/lib.rs @@ -29,9 +29,6 @@ macro_rules! chain_result { ($first_expr:expr, $($try_expr:expr),*) => ( $first_expr $(.and_then(|_| $try_expr))* ); - ($first_expr:expr, $($try_expr:expr),*,) => ( - chain_result! { $first_expr, $($try_expr),* } - ); } #[cfg(feature = "client")] diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index a0ccc8cc5..fd9221fe2 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -58,7 +58,7 @@ impl Multipart where R: HttpRequest { return Err(req); } - let boundary = req.boundary().unwrap().to_owned(); + let boundary = format!("\r\n--{}", req.boundary().unwrap()); debug!("Boundary: {}", boundary); @@ -300,7 +300,7 @@ impl<'a, R: HttpRequest + 'a> MultipartField<'a, R> { // The last two characters are "\r\n". // We can't do a simple trim because the content might be terminated // with line separators we want to preserve. - MultipartData::Text(&text[..text.len() - 2]) + MultipartData::Text(&text[..text.len()]) }, }; diff --git a/multipart/tests/local.rs b/multipart/tests/local.rs index 05a62affb..8b864d210 100644 --- a/multipart/tests/local.rs +++ b/multipart/tests/local.rs @@ -24,9 +24,17 @@ struct TestFields { #[test] fn local_test() { + env_logger::init().unwrap(); + let test_fields = gen_test_fields(); let buf = test_client(&test_fields); + + info!( + "\n--Test Buffer Begin--\n{}\n--Test Buffer End--", + String::from_utf8_lossy(&buf.buf) + ); + test_server(buf, test_fields); } @@ -58,8 +66,8 @@ fn gen_string() -> String { } fn gen_bytes() -> Vec { - const MIN_LEN: usize = 64; - const MAX_LEN: usize = 1024; + const MIN_LEN: usize = 8; + const MAX_LEN: usize = 32; let mut rng = rand::weak_rng(); let bytes_len = gen_range(MIN_LEN, MAX_LEN); @@ -78,7 +86,8 @@ fn test_client(test_fields: &TestFields) -> HttpBuffer { let mut test_files = test_fields.files.iter(); let mut multipart = Multipart::from_request(request).unwrap(); - + + // Intersperse file fields amongst text fields for (name, text) in &test_fields.texts { if let Some((file_name, file)) = test_files.next() { multipart = multipart.write_stream(file_name, &mut &**file, None, None); @@ -87,6 +96,12 @@ fn test_client(test_fields: &TestFields) -> HttpBuffer { multipart = multipart.write_text(name, text); } + // Write remaining files + for (file_name, file) in test_files { + multipart = multipart.write_stream(file_name, &mut &**file, None, None); + } + + multipart.send().unwrap() } @@ -103,7 +118,7 @@ fn test_server(buf: HttpBuffer, mut fields: TestFields) { assert!( text == test_text, "Expected {:?} for {:?} got {:?}", - text, field.name, test_text + test_text, field.name, text ); }, @@ -119,7 +134,7 @@ fn test_server(buf: HttpBuffer, mut fields: TestFields) { } assert!(fields.texts.is_empty(), "Text fields were not exhausted! Text fields: {:?}", fields.texts); - assert!(fields.files.is_empty(), "File fields were not exhausted!"); + assert!(fields.files.is_empty(), "File fields were not exhausted! File fields: {:?}", fields.files); } #[derive(Default, Debug)] From 055c77bafb757823084a93358301d08615a3f5d8 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Fri, 17 Jul 2015 05:23:36 -0700 Subject: [PATCH 077/453] Bump version --- multipart/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 8d37ed48f..224736d51 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "multipart" -version = "0.2.1" +version = "0.2.2" authors = ["Austin Bonander "] description = "An extension to the Hyper HTTP library that provides support for POST multipart/form-data requests on for both client and server." From c7cccd86c3763f9e52334bc64e15f6b7bbb732e0 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Fri, 17 Jul 2015 05:34:13 -0700 Subject: [PATCH 078/453] Remove TODO section --- multipart/README.md | 5 ----- 1 file changed, 5 deletions(-) diff --git a/multipart/README.md b/multipart/README.md index 03ac2feee..91eab505a 100644 --- a/multipart/README.md +++ b/multipart/README.md @@ -73,10 +73,5 @@ fn main() { ####[Documentation][2] -#####TODO: -- [ ] Add support for multiple files per field (nested boundaries) -- [ ] Expand integration tests to cover more cases - - [1]: https://github.com/hyperium/hyper [2]: http://rust-ci.org/cybergeek94/multipart/doc/multipart/ From 20040e72da099757052e2a545a69afdfdb4102ac Mon Sep 17 00:00:00 2001 From: Nicolas Cherel Date: Fri, 17 Jul 2015 17:18:02 +0200 Subject: [PATCH 079/453] fixes for boundary corner cases, beginning and end - in boundary.rs : just search for CRLF if there was chars before the boundary. Maybe this could be passed as a searched prefix to the BoundaryReader. - in mod.rs : removed CRLF prefix (see above), and added systematic line read after boundary to eat the CRLF, and check for multipart end --- multipart/src/server/boundary.rs | 4 ++++ multipart/src/server/mod.rs | 8 ++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/multipart/src/server/boundary.rs b/multipart/src/server/boundary.rs index b1018bc07..c2cb75f6e 100644 --- a/multipart/src/server/boundary.rs +++ b/multipart/src/server/boundary.rs @@ -40,6 +40,10 @@ impl BoundaryReader where R: Read { self.search_idx = last_search_idx + search_idx; if self.boundary_read { + // if boundary is preceded by CRLF, include it + if self.search_idx >= 2 && &buf[self.search_idx-2..self.search_idx] == b"\r\n" { + self.search_idx = self.search_idx - 2; + } break; } } diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index fd9221fe2..4bb8d8a4b 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -58,7 +58,7 @@ impl Multipart where R: HttpRequest { return Err(req); } - let boundary = format!("\r\n--{}", req.boundary().unwrap()); + let boundary = format!("--{}", req.boundary().unwrap()); debug!("Boundary: {}", boundary); @@ -81,11 +81,11 @@ impl Multipart where R: HttpRequest { pub fn read_entry(&mut self) -> io::Result>> { if self.at_end { return Ok(None); } - let mut byte2 = [0u8; 2]; try!(self.source.consume_boundary()); - try!(self.source.read(&mut byte2)); - self.at_end = &byte2 == b"--"; + self.at_end = { + try!(self.read_line()) == "--\r\n" + }; if !self.at_end { MultipartField::read_from(self) From dfbf43608855ae178fed400cb2dcffb5548db814 Mon Sep 17 00:00:00 2001 From: Nicolas Cherel Date: Fri, 17 Jul 2015 18:08:36 +0200 Subject: [PATCH 080/453] content type value string needs trimming before `parse::()` --- multipart/src/server/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index fd9221fe2..87f15307f 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -185,7 +185,7 @@ impl ContentType { debug!("Reading Content-Type header from line: {:?}", line); if let Some((cont_type, after_cont_type)) = get_str_after(CONTENT_TYPE, ';', line) { - let content_type = read_content_type(cont_type); + let content_type = read_content_type(cont_type.trim()); let boundary = get_str_after(BOUNDARY, '"', after_cont_type).map(|tup| tup.0.into()); @@ -195,7 +195,7 @@ impl ContentType { }) } else { get_remainder_after(CONTENT_TYPE, line).map(|cont_type| { - let content_type = read_content_type(cont_type); + let content_type = read_content_type(cont_type.trim()); ContentType { val: content_type, boundary: None } }) } From 8c36cfc74c5fd16b15de1256746bae351cae1123 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Fri, 17 Jul 2015 13:14:47 -0700 Subject: [PATCH 081/453] Bump version --- multipart/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 224736d51..1893de67c 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "multipart" -version = "0.2.2" +version = "0.2.3" authors = ["Austin Bonander "] description = "An extension to the Hyper HTTP library that provides support for POST multipart/form-data requests on for both client and server." From 442edb056a95bb9ac5730c21466a38de5436e6e7 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Fri, 17 Jul 2015 15:01:42 -0700 Subject: [PATCH 082/453] Update README.md --- multipart/README.md | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/multipart/README.md b/multipart/README.md index 91eab505a..66a4ff16b 100644 --- a/multipart/README.md +++ b/multipart/README.md @@ -1,10 +1,12 @@ -Multipart + Hyper [![Build Status](https://travis-ci.org/cybergeek94/multipart.svg?branch=master)](https://travis-ci.org/cybergeek94/multipart) +Multipart + Hyper [![Build Status](https://travis-ci.org/cybergeek94/multipart.svg?branch=master)](https://travis-ci.org/cybergeek94/multipart) [![On Crates.io](https://img.shields.io/crates/v/multipart.svg)](https://crates.io/crates/multipart) ========= Client- and server-side abstractions for HTTP file uploads (POST requests with `Content-Type: multipart/form-data`). Provides integration with [Hyper](https://github.com/hyperium/hyper) via the `hyper` feature. More to come! +####[Documentation](http://rust-ci.org/cybergeek94/multipart/doc/multipart/) + Usage ----- @@ -16,13 +18,13 @@ hyper = "*" url = "*" [dependencies.multipart] -version = "0.2.1" +version = "*" # Or use the version in the Crates.io badge above. # You can also select which features to compile: # default-features = false # features = ["hyper", "server", "client"] ``` -Client-side example using Hyper (`--features hyper,client` or default): +Client-side example using Hyper (`features = ["hyper", "client"]` or default): ```rust extern crate hyper; extern crate multipart; @@ -48,7 +50,7 @@ fn main() { } ``` -Server-side example using Hyper (`--features hyper,server` or default`): +Server-side example using Hyper (`features = ["hyper", "server"]` or default): ```rust use hyper::net::Fresh; use hyper::server::{Server, Request, Response}; @@ -71,7 +73,6 @@ fn main() { } ``` -####[Documentation][2] - -[1]: https://github.com/hyperium/hyper -[2]: http://rust-ci.org/cybergeek94/multipart/doc/multipart/ +License +------- +MIT From 79d138d53d61fa8d0507adf880407ac557610cf7 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Fri, 24 Jul 2015 11:15:08 -0700 Subject: [PATCH 083/453] Move local test to submodule of crate --- multipart/src/lib.rs | 3 + .../{tests/local.rs => src/local_test.rs} | 31 +-- multipart/src/server/boundary.rs | 189 +++++++++++++----- multipart/src/server/buf_read.rs | 89 +++++++++ multipart/src/server/mod.rs | 18 +- 5 files changed, 240 insertions(+), 90 deletions(-) rename multipart/{tests/local.rs => src/local_test.rs} (89%) create mode 100644 multipart/src/server/buf_read.rs diff --git a/multipart/src/lib.rs b/multipart/src/lib.rs index 8b29176df..314c7cfbc 100644 --- a/multipart/src/lib.rs +++ b/multipart/src/lib.rs @@ -36,6 +36,9 @@ pub mod client; #[cfg(feature = "server")] pub mod server; +#[cfg(all(test, feature = "client", feature = "server"))] +mod local_test; + const DIRNAME_LEN: usize = 12; fn temp_dir() -> PathBuf { diff --git a/multipart/tests/local.rs b/multipart/src/local_test.rs similarity index 89% rename from multipart/tests/local.rs rename to multipart/src/local_test.rs index 8b864d210..419dce58a 100644 --- a/multipart/tests/local.rs +++ b/multipart/src/local_test.rs @@ -1,14 +1,7 @@ -#[macro_use] -extern crate log; -extern crate env_logger; -extern crate rand; +use client::HttpRequest as ClientRequest; +use client::HttpStream as ClientStream; -extern crate multipart; - -use multipart::client::HttpRequest as ClientRequest; -use multipart::client::HttpStream as ClientStream; - -use multipart::server::HttpRequest as ServerRequest; +use server::HttpRequest as ServerRequest; use rand::Rng; use rand::distributions::{Range, Sample}; @@ -24,7 +17,7 @@ struct TestFields { #[test] fn local_test() { - env_logger::init().unwrap(); + let _ = ::env_logger::init(); let test_fields = gen_test_fields(); @@ -52,14 +45,14 @@ fn gen_test_fields() -> TestFields { } fn gen_range(min: usize, max: usize) -> usize { - Range::new(min, max).sample(&mut rand::weak_rng()) + Range::new(min, max).sample(&mut ::rand::weak_rng()) } fn gen_string() -> String { const MIN_LEN: usize = 3; const MAX_LEN: usize = 12; - let mut rng = rand::weak_rng(); + let mut rng = ::rand::weak_rng(); let str_len = gen_range(MIN_LEN, MAX_LEN); rng.gen_ascii_chars().take(str_len).collect() @@ -69,7 +62,7 @@ fn gen_bytes() -> Vec { const MIN_LEN: usize = 8; const MAX_LEN: usize = 32; - let mut rng = rand::weak_rng(); + let mut rng = ::rand::weak_rng(); let bytes_len = gen_range(MIN_LEN, MAX_LEN); let mut vec = vec![0u8; bytes_len]; @@ -79,7 +72,7 @@ fn gen_bytes() -> Vec { fn test_client(test_fields: &TestFields) -> HttpBuffer { - use multipart::client::Multipart; + use client::Multipart; let request = MockClientRequest::default(); @@ -106,7 +99,7 @@ fn test_client(test_fields: &TestFields) -> HttpBuffer { } fn test_server(buf: HttpBuffer, mut fields: TestFields) { - use multipart::server::{Multipart, MultipartData}; + use server::{Multipart, MultipartData}; let mut multipart = Multipart::from_request(buf.for_server()) .unwrap_or_else(|_| panic!("Buffer should be multipart!")); @@ -143,12 +136,6 @@ pub struct MockClientRequest { content_len: Option, } -impl MockClientRequest { - pub fn new() -> MockClientRequest { - Self::default() - } -} - impl ClientRequest for MockClientRequest { type Stream = HttpBuffer; type Error = io::Error; diff --git a/multipart/src/server/boundary.rs b/multipart/src/server/boundary.rs index c2cb75f6e..93c8ddc61 100644 --- a/multipart/src/server/boundary.rs +++ b/multipart/src/server/boundary.rs @@ -14,6 +14,7 @@ pub struct BoundaryReader { boundary: Vec, search_idx: usize, boundary_read: bool, + at_end: bool, } impl BoundaryReader where R: Read { @@ -24,49 +25,66 @@ impl BoundaryReader where R: Read { boundary: boundary.into(), search_idx: 0, boundary_read: false, + at_end: false, } } fn read_to_boundary(&mut self) -> io::Result<&[u8]> { + use log::LogLevel; + let buf = try!(self.buffer.fill_buf()); - if !self.boundary_read { - let last_search_idx = self.search_idx; - let lookahead_iter = buf[self.search_idx..].windows(self.boundary.len()).enumerate(); - - for (search_idx, maybe_boundary) in lookahead_iter { - if maybe_boundary[0] == self.boundary[0] { - self.boundary_read = self.boundary == maybe_boundary; - self.search_idx = last_search_idx + search_idx; - - if self.boundary_read { - // if boundary is preceded by CRLF, include it - if self.search_idx >= 2 && &buf[self.search_idx-2..self.search_idx] == b"\r\n" { - self.search_idx = self.search_idx - 2; - } - break; - } - } + while !(self.boundary_read || self.at_end) && self.search_idx < buf.len() { + let lookahead = &buf[self.search_idx..]; + + let safe_len = cmp::min(lookahead.len(), self.boundary.len()); + + if &lookahead[..safe_len] == &self.boundary[..safe_len] { + self.boundary_read = safe_len == self.boundary.len(); + break; } + + self.search_idx += 1; + } + + if self.search_idx >= 2 { + self.search_idx -= 2; } + debug!("Buf len: {} Search idx: {}", buf.len(), self.search_idx); - Ok(&buf[..self.search_idx]) + + if log_enabled!(LogLevel::Info) { + let _ = ::std::str::from_utf8(buf).map(|buf| + info!("Buf: {:?}", buf) + ); + } + + Ok(&buf[..self.search_idx]) } #[doc(hidden)] pub fn consume_boundary(&mut self) -> io::Result<()> { while !self.boundary_read { let buf_len = try!(self.read_to_boundary()).len(); + + if buf_len == 0 { + break; + } + self.consume(buf_len); } - let consume_amt = { - let boundary_len = self.boundary.len(); - let buf = try!(self.read_to_boundary()); - buf.len() + boundary_len - }; + let consume_amt = self.search_idx + self.boundary.len(); self.buffer.consume(consume_amt); + + let mut after_buf = [0u8; 2]; + // Substitute for Result::expect() being unstable. + self.buffer.read(&mut after_buf).unwrap_or_else(|err| + panic!("Failed to read 2 bytes after boundary. Reason: {:?}", err) + ); + self.at_end = &after_buf == b"--"; + self.search_idx = 0; self.boundary_read = false; @@ -128,46 +146,113 @@ fn copy_bytes(src: &[u8], dst: &mut [u8]) { } } -#[test] -fn test_boundary() { - const BOUNDARY: &'static str = "--boundary\r\n"; - const TEST_VAL: &'static str = "\r ---boundary\r +#[cfg(test)] +mod test { + use super::BoundaryReader; + + use std::cmp; + + use std::io; + use std::io::prelude::*; + + const BOUNDARY: &'static str = "--boundary"; + const TEST_VAL: &'static str = "--boundary\r dashed-value-1\r --boundary\r dashed-value-2\r ---boundary\r -"; +--boundary--"; + + #[test] + fn test_boundary() { + let _ = ::env_logger::init(); + debug!("Testing boundary (no split)"); - ::env_logger::init().unwrap(); + let src = &mut TEST_VAL.as_bytes(); + let reader = BoundaryReader::from_reader(src, BOUNDARY); + + test_boundary_reader(reader); + } + + struct SplitReader<'a> { + left: &'a [u8], + right: &'a [u8], + } + + impl<'a> SplitReader<'a> { + fn split(data: &'a [u8], at: usize) -> SplitReader<'a> { + let (left, right) = data.split_at(at); + + SplitReader { + left: left, + right: right, + } + } + } + + impl<'a> Read for SplitReader<'a> { + fn read(&mut self, dst: &mut [u8]) -> io::Result { + fn copy_bytes_partial(src: &mut &[u8], dst: &mut [u8]) -> usize { + let copy_amt = cmp::min(src.len(), dst.len()); + super::copy_bytes(&src[..copy_amt], dst); + *src = &src[copy_amt..]; + copy_amt + } - let src = &mut TEST_VAL.as_bytes(); - let mut reader = BoundaryReader::from_reader(src, BOUNDARY); + let mut copy_amt = copy_bytes_partial(&mut self.left, dst); - let ref mut buf = String::new(); + if copy_amt == 0 { + copy_amt = copy_bytes_partial(&mut self.right, dst) + }; - debug!("Read 1"); - let _ = reader.read_to_string(buf).unwrap(); - debug!("Buf: {:?}", buf); - assert!(buf.trim().is_empty()); + Ok(copy_amt) + } + } + + #[test] + fn test_split_boundary() { + let _ = ::env_logger::init(); + debug!("Testing boundary (split)"); + + // Substitute for `.step_by()` being unstable. + for split_at in (0 .. TEST_VAL.len()).filter(|x| x % 2 != 0) { + debug!("Testing split at: {}", split_at); + + let src = SplitReader::split(TEST_VAL.as_bytes(), split_at); + let reader = BoundaryReader::from_reader(src, BOUNDARY); + test_boundary_reader(reader); + } - buf.clear(); + } + + fn test_boundary_reader(mut reader: BoundaryReader) { + let ref mut buf = String::new(); + + debug!("Read 1"); + let _ = reader.read_to_string(buf).unwrap(); + assert!(buf.is_empty(), "Buffer not empty: {:?}", buf); + buf.clear(); - debug!("Consume 1"); - reader.consume_boundary().unwrap(); + debug!("Consume 1"); + reader.consume_boundary().unwrap(); - debug!("Read 2"); - let _ = reader.read_to_string(buf).unwrap(); - assert_eq!(buf.trim(), "dashed-value-1"); - buf.clear(); + debug!("Read 2"); + let _ = reader.read_to_string(buf).unwrap(); + assert_eq!(buf, "dashed-value-1"); + buf.clear(); - debug!("Consume 2"); - reader.consume_boundary().unwrap(); + debug!("Consume 2"); + reader.consume_boundary().unwrap(); - debug!("Read 3"); - let _ = reader.read_to_string(buf).unwrap(); - assert_eq!(buf.trim(), "dashed-value-2"); + debug!("Read 3"); + let _ = reader.read_to_string(buf).unwrap(); + assert_eq!(buf, "dashed-value-2"); + buf.clear(); - debug!("Consume 3"); - reader.consume_boundary().unwrap(); + debug!("Consume 3"); + reader.consume_boundary().unwrap(); + + debug!("Read 4"); + let _ = reader.read_to_string(buf).unwrap(); + assert!(buf.is_empty(), "Buffer not empty: {:?}", buf); + } } diff --git a/multipart/src/server/buf_read.rs b/multipart/src/server/buf_read.rs new file mode 100644 index 000000000..067206e84 --- /dev/null +++ b/multipart/src/server/buf_read.rs @@ -0,0 +1,89 @@ +use std::cmp; +use std::fmt; + +use std::io::prelude::*; +use std::io; + +const DEFAULT_BUF_SIZE: usize = 64 * 1024; + +/// A copy of `std::io::BufReader` with additional methods needed for multipart support. +pub struct CustomBufReader { + inner: R, + buf: Vec, + pos: usize, + cap: usize, +} + +impl CustomBufReader { + #[doc(hidden)] + pub fn new(inner: R) -> Self { + CustomBufReader::with_capacity(DEFAULT_BUF_SIZE, inner) + } + + #[doc(hidden)] + pub fn with_capacity(cap: usize, inner: R) -> Self { + CustomBufReader { + inner: inner, + buf: vec![0; cap], + pos: 0, + cap: 0, + } + } + + #[doc(hidden)] + pub fn fill_buf_min(&mut self, min: usize) -> io::Result<&[u8]> { + if self.pos == self.cap { + self.cap = try!(self.inner.read(&mut self.buf)); + self.pos = 0; + } else if min > self.cap - self.pos { + self.cap += try!(self.inner.read(&mut self.buf[self.cap..])); + } + + Ok(&self.buf[self.pos..self.cap]) + } + + #[doc(hidden)] + pub fn get_ref(&self) -> &R { &self.inner } +} + +impl Read for CustomBufReader { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + // If we don't have any buffered data and we're doing a massive read + // (larger than our internal buffer), bypass our internal buffer + // entirely. + if self.pos == self.cap && buf.len() >= self.buf.len() { + return self.inner.read(buf); + } + let nread = { + let mut rem = try!(self.fill_buf()); + try!(rem.read(buf)) + }; + self.consume(nread); + Ok(nread) + } +} + +impl BufRead for CustomBufReader { + fn fill_buf(&mut self) -> io::Result<&[u8]> { + // If we've reached the end of our internal buffer then we need to fetch + // some more data from the underlying reader. + if self.pos == self.cap { + self.cap = try!(self.inner.read(&mut self.buf)); + self.pos = 0; + } + Ok(&self.buf[self.pos..self.cap]) + } + + fn consume(&mut self, amt: usize) { + self.pos = cmp::min(self.pos + amt, self.cap); + } +} + +impl fmt::Debug for CustomBufReader where R: fmt::Debug { + fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + fmt.debug_struct("BufReader") + .field("reader", &self.inner) + .field("buffer", &format_args!("{}/{}", self.cap - self.pos, self.buf.len())) + .finish() + } +} diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index e12ca9237..9a634144c 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -5,7 +5,6 @@ //! to accept, parse, and serve HTTP `multipart/form-data` requests (file uploads). //! //! See the `Multipart` struct for more info. - use mime::Mime; use std::borrow::Borrow; @@ -18,9 +17,9 @@ use std::io::prelude::*; use std::path::{Path, PathBuf}; -#[doc(inline)] pub use self::boundary::BoundaryReader; +mod buf_read; mod boundary; #[cfg(feature = "hyper")] @@ -41,7 +40,6 @@ macro_rules! try_opt ( pub struct Multipart { source: BoundaryReader, line_buf: String, - at_end: bool, /// The directory for saving files in this request. /// By default, this is set to a subdirectory of `std::env::temp_dir()` with a /// random alphanumeric name. @@ -66,7 +64,6 @@ impl Multipart where R: HttpRequest { Multipart { source: BoundaryReader::from_reader(req, boundary), line_buf: String::new(), - at_end: false, save_dir: ::temp_dir(), } ) @@ -79,19 +76,8 @@ impl Multipart where R: HttpRequest { /// If the previously returned entry had contents of type `MultipartField::File`, /// calling this again will discard any unread contents of that entry. pub fn read_entry(&mut self) -> io::Result>> { - if self.at_end { return Ok(None); } - try!(self.source.consume_boundary()); - - self.at_end = { - try!(self.read_line()) == "--\r\n" - }; - - if !self.at_end { - MultipartField::read_from(self) - } else { - Ok(None) - } + MultipartField::read_from(self) } fn read_content_disposition(&mut self) -> io::Result> { From 2566b8e6edc4f19946ff06e03b84b093c8b6178d Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Fri, 24 Jul 2015 21:04:09 -0700 Subject: [PATCH 084/453] Fix boundary reading! --- multipart/Cargo.toml | 2 +- multipart/src/server/boundary.rs | 82 ++++++++++++++++++++++++-------- 2 files changed, 64 insertions(+), 20 deletions(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 1893de67c..ede594ce0 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "multipart" -version = "0.2.3" +version = "0.2.4" authors = ["Austin Bonander "] description = "An extension to the Hyper HTTP library that provides support for POST multipart/form-data requests on for both client and server." diff --git a/multipart/src/server/boundary.rs b/multipart/src/server/boundary.rs index 93c8ddc61..39f05d8f0 100644 --- a/multipart/src/server/boundary.rs +++ b/multipart/src/server/boundary.rs @@ -1,8 +1,9 @@ +use server::buf_read::CustomBufReader; + use std::cmp; use std::borrow::Borrow; use std::io; -use std::io::BufReader; use std::io::prelude::*; use std::ptr; @@ -10,7 +11,7 @@ use std::ptr; /// A struct implementing `Read` and `BufRead` that will yield bytes until it sees a given sequence. #[derive(Debug)] pub struct BoundaryReader { - buffer: BufReader, + buffer: CustomBufReader, boundary: Vec, search_idx: usize, boundary_read: bool, @@ -21,7 +22,7 @@ impl BoundaryReader where R: Read { #[doc(hidden)] pub fn from_reader>>(reader: R, boundary: B) -> BoundaryReader { BoundaryReader { - buffer: BufReader::new(reader), + buffer: CustomBufReader::new(reader), boundary: boundary.into(), search_idx: 0, boundary_read: false, @@ -32,7 +33,7 @@ impl BoundaryReader where R: Read { fn read_to_boundary(&mut self) -> io::Result<&[u8]> { use log::LogLevel; - let buf = try!(self.buffer.fill_buf()); + let buf = try!(self.buffer.fill_buf_min(self.boundary.len())); while !(self.boundary_read || self.at_end) && self.search_idx < buf.len() { let lookahead = &buf[self.search_idx..]; @@ -45,13 +46,12 @@ impl BoundaryReader where R: Read { } self.search_idx += 1; - } - - if self.search_idx >= 2 { - self.search_idx -= 2; } - debug!("Buf len: {} Search idx: {}", buf.len(), self.search_idx); + debug!( + "Buf len: {} Search idx: {} Boundary read: {:?}", + buf.len(), self.search_idx, self.boundary_read + ); if log_enabled!(LogLevel::Info) { let _ = ::std::str::from_utf8(buf).map(|buf| @@ -59,7 +59,11 @@ impl BoundaryReader where R: Read { ); } - Ok(&buf[..self.search_idx]) + if self.search_idx >= 2 { + Ok(&buf[..self.search_idx - 2]) + } else { + Ok(&buf[..self.search_idx]) + } } #[doc(hidden)] @@ -73,17 +77,51 @@ impl BoundaryReader where R: Read { self.consume(buf_len); } + - let consume_amt = self.search_idx + self.boundary.len(); + let consume_amt = { + let mut buf = try!(self.buffer.fill_buf_min(self.boundary.len() + 4)); + let mut consume_amt = self.boundary.len() + 2; + + if self.search_idx != 0 { + let (before, new_buf) = buf.split_at(2); + if before != b"\r\n" { + return Err(io::Error::new( + io::ErrorKind::Other, + format!("Expected {:?}, got {:?}", b"\r\n", &before) + )); + } + + buf = new_buf; + consume_amt += 2; + } - self.buffer.consume(consume_amt); + let (boundary, after) = buf.split_at(self.boundary.len()); + if &*self.boundary != boundary { + return Err(io::Error::new( + io::ErrorKind::Other, + format!("Expected {:?}, got {:?}", self.boundary, boundary) + )); + } - let mut after_buf = [0u8; 2]; - // Substitute for Result::expect() being unstable. - self.buffer.read(&mut after_buf).unwrap_or_else(|err| - panic!("Failed to read 2 bytes after boundary. Reason: {:?}", err) - ); - self.at_end = &after_buf == b"--"; + if &after[..2] != b"\r\n" { + self.at_end = after == b"--"; + + if !self.at_end { + return Err(io::Error::new( + io::ErrorKind::Other, + format!( + "Expected {:?} or {:?}, got {:?}", + b"\r\n", b"--", after + ) + )); + } + } + + consume_amt + }; + + self.buffer.consume(consume_amt); self.search_idx = 0; self.boundary_read = false; @@ -126,7 +164,13 @@ impl BufRead for BoundaryReader where R: Read { } fn consume(&mut self, amt: usize) { - let true_amt = cmp::min(amt, self.search_idx); + let mut search_idx = self.search_idx; + + if search_idx >= 2 { + search_idx -= 2; + } + + let true_amt = cmp::min(amt, search_idx); self.buffer.consume(true_amt); self.search_idx -= true_amt; } From 484505b5c920c7850fccea2c6a342c929ead715e Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Fri, 24 Jul 2015 22:02:09 -0700 Subject: [PATCH 085/453] Don't consume a boundary after the end --- multipart/src/server/boundary.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/multipart/src/server/boundary.rs b/multipart/src/server/boundary.rs index 39f05d8f0..c4afeb75b 100644 --- a/multipart/src/server/boundary.rs +++ b/multipart/src/server/boundary.rs @@ -68,6 +68,10 @@ impl BoundaryReader where R: Read { #[doc(hidden)] pub fn consume_boundary(&mut self) -> io::Result<()> { + if self.at_end { + return Ok(()); + } + while !self.boundary_read { let buf_len = try!(self.read_to_boundary()).len(); @@ -297,6 +301,9 @@ dashed-value-2\r debug!("Read 4"); let _ = reader.read_to_string(buf).unwrap(); - assert!(buf.is_empty(), "Buffer not empty: {:?}", buf); + assert!(buf.is_empty(), "Buffer not empty: {:?}", buf); + + debug!("Extra Consume"); + reader.consume_boundary().unwrap(); } } From 98b188e636d83dea348cf67019853b20ba2ff993 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sun, 26 Jul 2015 04:44:31 -0700 Subject: [PATCH 086/453] Fix building on Stable (until next release) --- multipart/src/server/buf_read.rs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/multipart/src/server/buf_read.rs b/multipart/src/server/buf_read.rs index 067206e84..c8a50f44e 100644 --- a/multipart/src/server/buf_read.rs +++ b/multipart/src/server/buf_read.rs @@ -81,9 +81,16 @@ impl BufRead for CustomBufReader { impl fmt::Debug for CustomBufReader where R: fmt::Debug { fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { - fmt.debug_struct("BufReader") + fmt.write_fmt(format_args!( + "CustomBufReader {{ reader: {:?}, buffer: {}/{}}}", + &self.inner, self.cap - self.pos, self.buf.len() + )) + + /* FIXME (07/26/15): Switch to this impl after the next Stable release. + fmt.debug_struct("CustomBufReader") .field("reader", &self.inner) - .field("buffer", &format_args!("{}/{}", self.cap - self.pos, self.buf.len())) + .field("buffer", &format_args!("{}/{}", )) .finish() + */ } } From a299f7c33c3556fcc48401542fe1cdbf7bd514cd Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sun, 26 Jul 2015 05:04:26 -0700 Subject: [PATCH 087/453] Remove superfluous method on `server::HttpRequest`. --- multipart/src/server/hyper.rs | 23 ++++++++--------------- multipart/src/server/mod.rs | 16 ++++++++-------- 2 files changed, 16 insertions(+), 23 deletions(-) diff --git a/multipart/src/server/hyper.rs b/multipart/src/server/hyper.rs index 484d8ceed..93dbbdc2d 100644 --- a/multipart/src/server/hyper.rs +++ b/multipart/src/server/hyper.rs @@ -62,24 +62,17 @@ where F: Fn(Multipart, Response), F: Send + Sync { } impl<'a, 'b> HttpRequest for Request<'a, 'b> { - fn is_multipart(&self) -> bool { - self.method == Method::Post && - self.headers.get::().map_or(false, |ct| { - let ContentType(ref mime) = *ct; - - debug!("Content-Type: {}", mime); - - match *mime { - Mime(TopLevel::Multipart, SubLevel::FormData, _) => true, - _ => false, - } - }) - } + fn multipart_boundary(&self) -> Option<&str> { + if self.method != Method::Post { + return None; + } - fn boundary(&self) -> Option<&str> { self.headers.get::().and_then(|ct| { let ContentType(ref mime) = *ct; - let Mime(_, _, ref params) = *mime; + let params = match *mime { + Mime(TopLevel::Multipart, SubLevel::FormData, ref params) => params, + _ => return None, + }; params.iter().find(|&&(ref name, _)| match *name { diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index 9a634144c..d5072b952 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -50,13 +50,11 @@ impl Multipart where R: HttpRequest { /// If the given `R: HttpRequest` is a POST request of `Content-Type: multipart/form-data`, /// return the wrapped request as `Ok(Multipart)`, otherwise `Err(R)`. pub fn from_request(req: R) -> Result, R> { - if !req.is_multipart() { return Err(req); } - - if req.boundary().is_none() { + if req.multipart_boundary().is_none() { return Err(req); } - let boundary = format!("--{}", req.boundary().unwrap()); + let boundary = format!("--{}", req.multipart_boundary().unwrap()); debug!("Boundary: {}", boundary); @@ -245,10 +243,12 @@ fn get_remainder_after<'a>(needle: &str, haystack: &'a str) -> Option<(&'a str)> /// A server-side HTTP request that may or may not be multipart. pub trait HttpRequest: Read { - /// Return `true` if this request is a `multipart/form-data` request, `false` otherwise. - fn is_multipart(&self) -> bool; - /// Get the boundary string of this request if it is `multipart/form-data`. - fn boundary(&self) -> Option<&str>; + /// Get the boundary string of this request if it is a POST request + /// with the `Content-Type` header set to `multipart/form-data`. + /// + /// The boundary string should be supplied as an extra value of the `Content-Type` header, e.g. + /// `Content-Type: multipart/form-data; boundary=myboundary`. + fn multipart_boundary(&self) -> Option<&str>; } /// A field in a multipart request. May be either text or a binary stream (file). From 9d4b46a97266b1046d055795d720feb937d18be9 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sun, 26 Jul 2015 05:10:53 -0700 Subject: [PATCH 088/453] Fix client API nit, fix test --- multipart/src/client/mod.rs | 8 ++++---- multipart/src/local_test.rs | 9 ++++----- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/multipart/src/client/mod.rs b/multipart/src/client/mod.rs index e10c50841..4ca770ca6 100644 --- a/multipart/src/client/mod.rs +++ b/multipart/src/client/mod.rs @@ -72,7 +72,7 @@ impl Multipart { /// /// ##Errors /// If something went wrong with the HTTP stream. - pub fn write_text, V: AsRef>(mut self, name: N, val: V) -> Self { + pub fn write_text, V: AsRef>(&mut self, name: N, val: V) -> &mut Self { if self.last_err.is_none() { self.last_err = chain_result! { self.write_field_headers(name.as_ref(), None, None), @@ -94,7 +94,7 @@ impl Multipart { /// ##Errors /// If there was a problem opening the file (was a directory or didn't exist), /// or if something went wrong with the HTTP stream. - pub fn write_file, P: AsRef>(mut self, name: N, path: P) -> Self { + pub fn write_file, P: AsRef>(&mut self, name: N, path: P) -> &mut Self { if self.last_err.is_none() { let path = path.as_ref(); @@ -132,8 +132,8 @@ impl Multipart { /// If the reader returned an error, or if something went wrong with the HTTP stream. // RFC: How to format this declaration? pub fn write_stream, St: Read>( - mut self, name: N, read: &mut St, filename: Option<&str>, content_type: Option - ) -> Self { + &mut self, name: N, read: &mut St, filename: Option<&str>, content_type: Option + ) -> &mut Self { if self.last_err.is_none() { let content_type = content_type.unwrap_or_else(::mime_guess::octet_stream); diff --git a/multipart/src/local_test.rs b/multipart/src/local_test.rs index 419dce58a..2585558f1 100644 --- a/multipart/src/local_test.rs +++ b/multipart/src/local_test.rs @@ -83,15 +83,15 @@ fn test_client(test_fields: &TestFields) -> HttpBuffer { // Intersperse file fields amongst text fields for (name, text) in &test_fields.texts { if let Some((file_name, file)) = test_files.next() { - multipart = multipart.write_stream(file_name, &mut &**file, None, None); + multipart.write_stream(file_name, &mut &**file, None, None); } - multipart = multipart.write_text(name, text); + multipart.write_text(name, text); } // Write remaining files for (file_name, file) in test_files { - multipart = multipart.write_stream(file_name, &mut &**file, None, None); + multipart.write_stream(file_name, &mut &**file, None, None); } @@ -209,7 +209,6 @@ impl<'a> Read for ServerBuffer<'a> { } impl<'a> ServerRequest for ServerBuffer<'a> { - fn is_multipart(&self) -> bool { true } - fn boundary(&self) -> Option<&str> { Some(&self.boundary) } + fn multipart_boundary(&self) -> Option<&str> { Some(&self.boundary) } } From bd31010b7cdc2b7a9c25f1a88c25c3999522075e Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sun, 26 Jul 2015 05:11:25 -0700 Subject: [PATCH 089/453] Bump major version to `0.3.0` --- multipart/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index ede594ce0..86d92c951 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "multipart" -version = "0.2.4" +version = "0.3.0" authors = ["Austin Bonander "] description = "An extension to the Hyper HTTP library that provides support for POST multipart/form-data requests on for both client and server." From 00678d9429a290a6f191dc35235540b75e49930d Mon Sep 17 00:00:00 2001 From: Nicolas Cherel Date: Mon, 27 Jul 2015 11:07:04 +0200 Subject: [PATCH 090/453] =?UTF-8?q?compare=20with=20ending=20two=20first?= =?UTF-8?q?=20characters=20since=20it=20can=20contain=20=E2=80=98--\r\n?= =?UTF-8?q?=E2=80=99=20(boundary.len()+4).?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Just to mention it, I don’t know if something should be done for the ‘\r\n’ after the boundary ending. Since content after boundary is supposed to be ignored, I don’t think it needs to be enforced, and does not need any care about being consumed or not. --- multipart/src/server/boundary.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/src/server/boundary.rs b/multipart/src/server/boundary.rs index c4afeb75b..474fcc00f 100644 --- a/multipart/src/server/boundary.rs +++ b/multipart/src/server/boundary.rs @@ -109,7 +109,7 @@ impl BoundaryReader where R: Read { } if &after[..2] != b"\r\n" { - self.at_end = after == b"--"; + self.at_end = &after[..2] == b"--"; if !self.at_end { return Err(io::Error::new( From 0af51a57e713e7939fa0cc1b5e3d680077dee539 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Mon, 27 Jul 2015 15:42:42 -0700 Subject: [PATCH 091/453] Bump version # --- multipart/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 86d92c951..02a5bccda 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "multipart" -version = "0.3.0" +version = "0.3.1" authors = ["Austin Bonander "] description = "An extension to the Hyper HTTP library that provides support for POST multipart/form-data requests on for both client and server." From 2830f9aa74c5c3b4391743eacb1738a9103ca769 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Thu, 20 Aug 2015 11:49:41 -0700 Subject: [PATCH 092/453] Update docs upload token --- multipart/.travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/.travis.yml b/multipart/.travis.yml index 817238672..46f6f07cb 100644 --- a/multipart/.travis.yml +++ b/multipart/.travis.yml @@ -8,7 +8,7 @@ os: - osx env: global: - - secure: eJpqXoY/xuCQRYmLBK9dLLzTGHvsBWe67wRN0fljesRgSnU+H09OsWI22DB9z/e6pMKjZpfDuyRkJaHVj8N7rwEfSeKdywv6uKqs5/Mi7dVINnzVA7jU3E4kLc+EuASF8w5d85kfccGNGs7qM0Uwz/i4da3Gw4B+cSc5cqjWsIY= + - secure: UrIkXI/5PqRVn9GeCAtadkvIcnGY9+uMKfz50099M+ncf7/ZAwL5T6AjMmV9FqgvRShpW7px8Aj7uEGwbNgUGIuiYENWeoAi7LAh5OatUHBMVNFLnCLxHtnmj8G3RgNAql5wiEvvnaWvb+joi8zNJ4lJ5m6kkrAPxwx1Uj6uYmw= after_success: - | test ${TRAVIS_PULL_REQUEST} == "false" && \ From c4529eed3f2a2e17e01b5d033bbe7e6c6acd3cbb Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sun, 7 Feb 2016 03:44:01 -0800 Subject: [PATCH 093/453] Use `buf_redux`, eliminate unsafe code in server --- multipart/Cargo.toml | 8 ++- multipart/src/server/boundary.rs | 57 ++++++++----------- multipart/src/server/buf_read.rs | 96 -------------------------------- multipart/src/server/mod.rs | 1 - 4 files changed, 28 insertions(+), 134 deletions(-) delete mode 100644 multipart/src/server/buf_read.rs diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 02a5bccda..55ee70bed 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -4,7 +4,7 @@ name = "multipart" version = "0.3.1" authors = ["Austin Bonander "] -description = "An extension to the Hyper HTTP library that provides support for POST multipart/form-data requests on for both client and server." +description = "A backend-agnostic extension for HTTP libraries that provides support for POST multipart/form-data requests on for both client and server." repository = "http://github.com/cybergeek94/multipart" @@ -15,7 +15,7 @@ license = "MIT" keywords = ["form-data", "hyper", "http", "post", "upload"] [features] -server = [] +server = ["buf_redux"] client = [] default = ["hyper", "server", "client"] @@ -30,6 +30,10 @@ rand = "*" version = "*" optional = true +[dependencies.buf_redux] +version = "0.1" +optional = true + [dev-dependencies.hyper] version = "*" # Disable SSL for testing purposes diff --git a/multipart/src/server/boundary.rs b/multipart/src/server/boundary.rs index 474fcc00f..35c4df7b8 100644 --- a/multipart/src/server/boundary.rs +++ b/multipart/src/server/boundary.rs @@ -1,4 +1,6 @@ -use server::buf_read::CustomBufReader; +extern crate buf_redux; + +use self::buf_redux::BufReader; use std::cmp; use std::borrow::Borrow; @@ -6,12 +8,10 @@ use std::borrow::Borrow; use std::io; use std::io::prelude::*; -use std::ptr; - /// A struct implementing `Read` and `BufRead` that will yield bytes until it sees a given sequence. #[derive(Debug)] pub struct BoundaryReader { - buffer: CustomBufReader, + buf: BufReader, boundary: Vec, search_idx: usize, boundary_read: bool, @@ -22,7 +22,7 @@ impl BoundaryReader where R: Read { #[doc(hidden)] pub fn from_reader>>(reader: R, boundary: B) -> BoundaryReader { BoundaryReader { - buffer: CustomBufReader::new(reader), + buf: BufReader::new(reader), boundary: boundary.into(), search_idx: 0, boundary_read: false, @@ -33,7 +33,7 @@ impl BoundaryReader where R: Read { fn read_to_boundary(&mut self) -> io::Result<&[u8]> { use log::LogLevel; - let buf = try!(self.buffer.fill_buf_min(self.boundary.len())); + let buf = try!(fill_buf_min(&mut self.buf, self.boundary.len())); while !(self.boundary_read || self.at_end) && self.search_idx < buf.len() { let lookahead = &buf[self.search_idx..]; @@ -84,7 +84,7 @@ impl BoundaryReader where R: Read { let consume_amt = { - let mut buf = try!(self.buffer.fill_buf_min(self.boundary.len() + 4)); + let mut buf = try!(fill_buf_min(&mut self.buf, self.boundary.len() + 4)); let mut consume_amt = self.boundary.len() + 2; if self.search_idx != 0 { @@ -125,7 +125,7 @@ impl BoundaryReader where R: Read { consume_amt }; - self.buffer.consume(consume_amt); + self.buf.consume(consume_amt); self.search_idx = 0; self.boundary_read = false; @@ -143,22 +143,20 @@ impl BoundaryReader where R: Read { impl Borrow for BoundaryReader { fn borrow(&self) -> &R { - self.buffer.get_ref() + self.buf.get_ref() } } impl Read for BoundaryReader where R: Read { fn read(&mut self, out: &mut [u8]) -> io::Result { - let consume_len = { - let buf = try!(self.read_to_boundary()); - let trunc_len = cmp::min(buf.len(), out.len()); - copy_bytes(&buf[..trunc_len], out); - trunc_len + let read = { + let mut buf = try!(self.read_to_boundary()); + // This shouldn't ever be an error so unwrapping is fine. + buf.read(out).unwrap() }; - self.consume(consume_len); - - Ok(consume_len) + self.consume(read); + Ok(read) } } @@ -175,31 +173,23 @@ impl BufRead for BoundaryReader where R: Read { } let true_amt = cmp::min(amt, search_idx); - self.buffer.consume(true_amt); + self.buf.consume(true_amt); self.search_idx -= true_amt; } } -// copied from `std::slice::bytes` due to unstable -fn copy_bytes(src: &[u8], dst: &mut [u8]) { - let len_src = src.len(); - assert!(dst.len() >= len_src); - // `dst` is unaliasable, so we know statically it doesn't overlap with `src`. - unsafe { - ptr::copy_nonoverlapping( - src.as_ptr(), - dst.as_mut_ptr(), - len_src - ); +fn fill_buf_min(buf: &mut BufReader, min: usize) -> io::Result<&[u8]> { + if buf.available() < min { + try!(buf.read_into_buf()); } + + Ok(buf.get_buf()) } #[cfg(test)] mod test { use super::BoundaryReader; - use std::cmp; - use std::io; use std::io::prelude::*; @@ -240,10 +230,7 @@ dashed-value-2\r impl<'a> Read for SplitReader<'a> { fn read(&mut self, dst: &mut [u8]) -> io::Result { fn copy_bytes_partial(src: &mut &[u8], dst: &mut [u8]) -> usize { - let copy_amt = cmp::min(src.len(), dst.len()); - super::copy_bytes(&src[..copy_amt], dst); - *src = &src[copy_amt..]; - copy_amt + src.read(dst).unwrap() } let mut copy_amt = copy_bytes_partial(&mut self.left, dst); diff --git a/multipart/src/server/buf_read.rs b/multipart/src/server/buf_read.rs deleted file mode 100644 index c8a50f44e..000000000 --- a/multipart/src/server/buf_read.rs +++ /dev/null @@ -1,96 +0,0 @@ -use std::cmp; -use std::fmt; - -use std::io::prelude::*; -use std::io; - -const DEFAULT_BUF_SIZE: usize = 64 * 1024; - -/// A copy of `std::io::BufReader` with additional methods needed for multipart support. -pub struct CustomBufReader { - inner: R, - buf: Vec, - pos: usize, - cap: usize, -} - -impl CustomBufReader { - #[doc(hidden)] - pub fn new(inner: R) -> Self { - CustomBufReader::with_capacity(DEFAULT_BUF_SIZE, inner) - } - - #[doc(hidden)] - pub fn with_capacity(cap: usize, inner: R) -> Self { - CustomBufReader { - inner: inner, - buf: vec![0; cap], - pos: 0, - cap: 0, - } - } - - #[doc(hidden)] - pub fn fill_buf_min(&mut self, min: usize) -> io::Result<&[u8]> { - if self.pos == self.cap { - self.cap = try!(self.inner.read(&mut self.buf)); - self.pos = 0; - } else if min > self.cap - self.pos { - self.cap += try!(self.inner.read(&mut self.buf[self.cap..])); - } - - Ok(&self.buf[self.pos..self.cap]) - } - - #[doc(hidden)] - pub fn get_ref(&self) -> &R { &self.inner } -} - -impl Read for CustomBufReader { - fn read(&mut self, buf: &mut [u8]) -> io::Result { - // If we don't have any buffered data and we're doing a massive read - // (larger than our internal buffer), bypass our internal buffer - // entirely. - if self.pos == self.cap && buf.len() >= self.buf.len() { - return self.inner.read(buf); - } - let nread = { - let mut rem = try!(self.fill_buf()); - try!(rem.read(buf)) - }; - self.consume(nread); - Ok(nread) - } -} - -impl BufRead for CustomBufReader { - fn fill_buf(&mut self) -> io::Result<&[u8]> { - // If we've reached the end of our internal buffer then we need to fetch - // some more data from the underlying reader. - if self.pos == self.cap { - self.cap = try!(self.inner.read(&mut self.buf)); - self.pos = 0; - } - Ok(&self.buf[self.pos..self.cap]) - } - - fn consume(&mut self, amt: usize) { - self.pos = cmp::min(self.pos + amt, self.cap); - } -} - -impl fmt::Debug for CustomBufReader where R: fmt::Debug { - fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { - fmt.write_fmt(format_args!( - "CustomBufReader {{ reader: {:?}, buffer: {}/{}}}", - &self.inner, self.cap - self.pos, self.buf.len() - )) - - /* FIXME (07/26/15): Switch to this impl after the next Stable release. - fmt.debug_struct("CustomBufReader") - .field("reader", &self.inner) - .field("buffer", &format_args!("{}/{}", )) - .finish() - */ - } -} diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index d5072b952..bbded8f31 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -19,7 +19,6 @@ use std::path::{Path, PathBuf}; pub use self::boundary::BoundaryReader; -mod buf_read; mod boundary; #[cfg(feature = "hyper")] From 4c408a61a3404111113fe6ced7b92e7b807c9283 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sun, 7 Feb 2016 16:51:17 -0800 Subject: [PATCH 094/453] Improvements to server-side parsing --- multipart/Cargo.toml | 11 ++- multipart/src/server/boundary.rs | 119 ++++++++++++------------------- multipart/src/server/mod.rs | 24 ++++++- 3 files changed, 74 insertions(+), 80 deletions(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 55ee70bed..4bffa6da4 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "multipart" -version = "0.3.1" +version = "0.4" authors = ["Austin Bonander "] description = "A backend-agnostic extension for HTTP libraries that provides support for POST multipart/form-data requests on for both client and server." @@ -15,7 +15,7 @@ license = "MIT" keywords = ["form-data", "hyper", "http", "post", "upload"] [features] -server = ["buf_redux"] +server = ["buf_redux", "memchr"] client = [] default = ["hyper", "server", "client"] @@ -34,7 +34,6 @@ optional = true version = "0.1" optional = true -[dev-dependencies.hyper] -version = "*" -# Disable SSL for testing purposes -default-features = false +[dependencies.memchr] +version = "0.1" +optional = true diff --git a/multipart/src/server/boundary.rs b/multipart/src/server/boundary.rs index 35c4df7b8..9f84fc3d8 100644 --- a/multipart/src/server/boundary.rs +++ b/multipart/src/server/boundary.rs @@ -1,6 +1,8 @@ extern crate buf_redux; +extern crate memchr; use self::buf_redux::BufReader; +use self::memchr::memchr; use std::cmp; use std::borrow::Borrow; @@ -35,35 +37,47 @@ impl BoundaryReader where R: Read { let buf = try!(fill_buf_min(&mut self.buf, self.boundary.len())); + debug!( + "Before-loop Buf len: {} Search idx: {} Boundary read: {:?}", + buf.len(), self.search_idx, self.boundary_read + ); + while !(self.boundary_read || self.at_end) && self.search_idx < buf.len() { let lookahead = &buf[self.search_idx..]; - let safe_len = cmp::min(lookahead.len(), self.boundary.len()); + let maybe_boundary = memchr(self.boundary[0], lookahead); - if &lookahead[..safe_len] == &self.boundary[..safe_len] { - self.boundary_read = safe_len == self.boundary.len(); - break; - } + debug!("maybe_boundary: {:?}", maybe_boundary); - self.search_idx += 1; - } + self.search_idx = match maybe_boundary { + Some(boundary_start) => self.search_idx + boundary_start, + None => buf.len(), + }; + if self.search_idx + self.boundary.len() <= buf.len() { + let test = &buf[self.search_idx .. self.search_idx + self.boundary.len()]; + + match first_nonmatching_idx(test, &self.boundary) { + Some(idx) => self.search_idx += idx, + None => self.boundary_read = true, + } + } else { + break; + } + } + debug!( - "Buf len: {} Search idx: {} Boundary read: {:?}", + "After-loop Buf len: {} Search idx: {} Boundary read: {:?}", buf.len(), self.search_idx, self.boundary_read - ); + ); if log_enabled!(LogLevel::Info) { let _ = ::std::str::from_utf8(buf).map(|buf| info!("Buf: {:?}", buf) ); } - - if self.search_idx >= 2 { - Ok(&buf[..self.search_idx - 2]) - } else { - Ok(&buf[..self.search_idx]) - } + + Ok(&buf[..self.search_idx]) } #[doc(hidden)] @@ -81,51 +95,8 @@ impl BoundaryReader where R: Read { self.consume(buf_len); } - - - let consume_amt = { - let mut buf = try!(fill_buf_min(&mut self.buf, self.boundary.len() + 4)); - let mut consume_amt = self.boundary.len() + 2; - - if self.search_idx != 0 { - let (before, new_buf) = buf.split_at(2); - if before != b"\r\n" { - return Err(io::Error::new( - io::ErrorKind::Other, - format!("Expected {:?}, got {:?}", b"\r\n", &before) - )); - } - - buf = new_buf; - consume_amt += 2; - } - - let (boundary, after) = buf.split_at(self.boundary.len()); - if &*self.boundary != boundary { - return Err(io::Error::new( - io::ErrorKind::Other, - format!("Expected {:?}, got {:?}", self.boundary, boundary) - )); - } - - if &after[..2] != b"\r\n" { - self.at_end = &after[..2] == b"--"; - - if !self.at_end { - return Err(io::Error::new( - io::ErrorKind::Other, - format!( - "Expected {:?} or {:?}, got {:?}", - b"\r\n", b"--", after - ) - )); - } - } - - consume_amt - }; - self.buf.consume(consume_amt); + self.buf.consume(self.boundary.len()); self.search_idx = 0; self.boundary_read = false; @@ -166,13 +137,10 @@ impl BufRead for BoundaryReader where R: Read { } fn consume(&mut self, amt: usize) { - let mut search_idx = self.search_idx; + let true_amt = cmp::min(amt, self.search_idx); - if search_idx >= 2 { - search_idx -= 2; - } + debug!("Consume! amt: {} true amt: {}", amt, true_amt); - let true_amt = cmp::min(amt, search_idx); self.buf.consume(true_amt); self.search_idx -= true_amt; } @@ -186,6 +154,16 @@ fn fill_buf_min(buf: &mut BufReader, min: usize) -> io::Result<&[u8] Ok(buf.get_buf()) } +fn first_nonmatching_idx(left: &[u8], right: &[u8]) -> Option { + for (idx, (lb, rb)) in left.iter().zip(right).enumerate() { + if lb != rb { + return Some(idx); + } + } + + None +} + #[cfg(test)] mod test { use super::BoundaryReader; @@ -193,8 +171,8 @@ mod test { use std::io; use std::io::prelude::*; - const BOUNDARY: &'static str = "--boundary"; - const TEST_VAL: &'static str = "--boundary\r + const BOUNDARY: &'static str = "\r\n--boundary"; + const TEST_VAL: &'static str = "\r\n--boundary\r dashed-value-1\r --boundary\r dashed-value-2\r @@ -272,7 +250,7 @@ dashed-value-2\r debug!("Read 2"); let _ = reader.read_to_string(buf).unwrap(); - assert_eq!(buf, "dashed-value-1"); + assert_eq!(buf, "\r\ndashed-value-1"); buf.clear(); debug!("Consume 2"); @@ -280,7 +258,7 @@ dashed-value-2\r debug!("Read 3"); let _ = reader.read_to_string(buf).unwrap(); - assert_eq!(buf, "dashed-value-2"); + assert_eq!(buf, "\r\ndashed-value-2"); buf.clear(); debug!("Consume 3"); @@ -288,9 +266,6 @@ dashed-value-2\r debug!("Read 4"); let _ = reader.read_to_string(buf).unwrap(); - assert!(buf.is_empty(), "Buffer not empty: {:?}", buf); - - debug!("Extra Consume"); - reader.consume_boundary().unwrap(); + assert_eq!(buf, "--"); } } diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index bbded8f31..d08c7e67e 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -53,7 +53,7 @@ impl Multipart where R: HttpRequest { return Err(req); } - let boundary = format!("--{}", req.multipart_boundary().unwrap()); + let boundary = format!("\r\n--{}", req.multipart_boundary().unwrap()); debug!("Boundary: {}", boundary); @@ -73,7 +73,10 @@ impl Multipart where R: HttpRequest { /// If the previously returned entry had contents of type `MultipartField::File`, /// calling this again will discard any unread contents of that entry. pub fn read_entry(&mut self) -> io::Result>> { - try!(self.source.consume_boundary()); + if !try!(self.consume_boundary()) { + return Ok(None); + } + MultipartField::read_from(self) } @@ -146,6 +149,23 @@ impl Multipart where R: HttpRequest { Err(err) => Err(err), } } + + fn consume_boundary(&mut self) -> io::Result { + try!(self.source.consume_boundary()); + + let mut out = [0; 2]; + let _ = try!(self.source.read(&mut out)); + + if *b"\r\n" == out { + return Ok(true); + } else { + if *b"--" != out { + warn!("Unexpected 2-bytes after boundary: {:?}", out); + } + + return Ok(false); + } + } } impl Borrow for Multipart where R: HttpRequest { From 2deebb2f9ce79608a3a1da61924c21f9ff6c6b41 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sun, 7 Feb 2016 16:51:55 -0800 Subject: [PATCH 095/453] Fix crate version --- multipart/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 4bffa6da4..c93a7bb12 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "multipart" -version = "0.4" +version = "0.4.0" authors = ["Austin Bonander "] description = "A backend-agnostic extension for HTTP libraries that provides support for POST multipart/form-data requests on for both client and server." From 06cebdb005304643df46ef0937bcb11ad3263311 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sun, 7 Feb 2016 16:55:56 -0800 Subject: [PATCH 096/453] Cement version numbers for Crates.io --- multipart/Cargo.toml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index c93a7bb12..b1c138a7a 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -20,14 +20,14 @@ client = [] default = ["hyper", "server", "client"] [dependencies] -log = "*" -env_logger = "*" -mime = "*" -mime_guess = "*" -rand = "*" +log = "0.3" +env_logger = "0.3" +mime = "0.1" +mime_guess = "1.4" +rand = "0.3" [dependencies.hyper] -version = "*" +version = "0.7" optional = true [dependencies.buf_redux] From 0f913e506ebf1320cfd58bd5902ed56b64c0105d Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Thu, 11 Feb 2016 15:56:54 -0800 Subject: [PATCH 097/453] Fix logic errors in boundary reading --- multipart/src/local_test.rs | 22 +++++++++---------- multipart/src/server/boundary.rs | 36 ++++++++++++++++++++++++-------- multipart/src/server/mod.rs | 2 +- 3 files changed, 39 insertions(+), 21 deletions(-) diff --git a/multipart/src/local_test.rs b/multipart/src/local_test.rs index 2585558f1..d082cdc9d 100644 --- a/multipart/src/local_test.rs +++ b/multipart/src/local_test.rs @@ -23,7 +23,7 @@ fn local_test() { let buf = test_client(&test_fields); - info!( + trace!( "\n--Test Buffer Begin--\n{}\n--Test Buffer End--", String::from_utf8_lossy(&buf.buf) ); @@ -33,7 +33,7 @@ fn local_test() { fn gen_test_fields() -> TestFields { const MIN_FIELDS: usize = 1; - const MAX_FIELDS: usize = 5; + const MAX_FIELDS: usize = 3; let texts_count = gen_range(MIN_FIELDS, MAX_FIELDS); let files_count = gen_range(MIN_FIELDS, MAX_FIELDS); @@ -50,7 +50,7 @@ fn gen_range(min: usize, max: usize) -> usize { fn gen_string() -> String { const MIN_LEN: usize = 3; - const MAX_LEN: usize = 12; + const MAX_LEN: usize = 8; let mut rng = ::rand::weak_rng(); let str_len = gen_range(MIN_LEN, MAX_LEN); @@ -60,17 +60,15 @@ fn gen_string() -> String { fn gen_bytes() -> Vec { const MIN_LEN: usize = 8; - const MAX_LEN: usize = 32; + const MAX_LEN: usize = 16; let mut rng = ::rand::weak_rng(); let bytes_len = gen_range(MIN_LEN, MAX_LEN); - let mut vec = vec![0u8; bytes_len]; - rng.fill_bytes(&mut vec); - vec + rng.gen_ascii_chars().take(bytes_len) + .map(|c| c as u8).collect() } - fn test_client(test_fields: &TestFields) -> HttpBuffer { use client::Multipart; @@ -110,8 +108,8 @@ fn test_server(buf: HttpBuffer, mut fields: TestFields) { let test_text = fields.texts.remove(&field.name).unwrap(); assert!( text == test_text, - "Expected {:?} for {:?} got {:?}", - test_text, field.name, text + "Unexpected data for field {:?}: Expected {:?}, got {:?}", + field.name, test_text, text ); }, @@ -121,7 +119,9 @@ fn test_server(buf: HttpBuffer, mut fields: TestFields) { let mut bytes = Vec::with_capacity(test_bytes.len()); file.read_to_end(&mut bytes).unwrap(); - assert!(bytes == test_bytes, "Unexpected data for {:?}", field.name); + assert!(bytes == test_bytes, "Unexpected data for file {:?}: Expected {:?}, Got {:?}", + field.name, String::from_utf8_lossy(&test_bytes), String::from_utf8_lossy(&bytes) + ); }, } } diff --git a/multipart/src/server/boundary.rs b/multipart/src/server/boundary.rs index 9f84fc3d8..f7294988d 100644 --- a/multipart/src/server/boundary.rs +++ b/multipart/src/server/boundary.rs @@ -36,6 +36,10 @@ impl BoundaryReader where R: Read { use log::LogLevel; let buf = try!(fill_buf_min(&mut self.buf, self.boundary.len())); + + if log_enabled!(LogLevel::Trace) { + trace!("Buf: {:?}", String::from_utf8_lossy(buf)); + } debug!( "Before-loop Buf len: {} Search idx: {} Boundary read: {:?}", @@ -64,20 +68,34 @@ impl BoundaryReader where R: Read { } else { break; } - } + } debug!( "After-loop Buf len: {} Search idx: {} Boundary read: {:?}", buf.len(), self.search_idx, self.boundary_read - ); + ); + + + let mut buf_end = self.search_idx; - if log_enabled!(LogLevel::Info) { - let _ = ::std::str::from_utf8(buf).map(|buf| - info!("Buf: {:?}", buf) - ); + if self.boundary_read && self.search_idx >= 2 { + let two_bytes_before = &buf[self.search_idx - 2 .. self.search_idx]; + + debug!("Two bytes before: {:?} (\"\\r\\n\": {:?})", two_bytes_before, b"\r\n"); + + if two_bytes_before == &*b"\r\n" { + debug!("Subtract two!"); + buf_end -= 2; + } } - - Ok(&buf[..self.search_idx]) + + let ret_buf = &buf[..buf_end]; + + if log_enabled!(LogLevel::Trace) { + trace!("Returning buf: {:?}", String::from_utf8_lossy(ret_buf)); + } + + Ok(ret_buf) } #[doc(hidden)] @@ -96,7 +114,7 @@ impl BoundaryReader where R: Read { self.consume(buf_len); } - self.buf.consume(self.boundary.len()); + self.buf.consume(self.search_idx + self.boundary.len()); self.search_idx = 0; self.boundary_read = false; diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index d08c7e67e..b5632f513 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -53,7 +53,7 @@ impl Multipart where R: HttpRequest { return Err(req); } - let boundary = format!("\r\n--{}", req.multipart_boundary().unwrap()); + let boundary = format!("--{}", req.multipart_boundary().unwrap()); debug!("Boundary: {}", boundary); From bd397ccc3c30e1064d71eec970f9a70711fa1365 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Thu, 11 Feb 2016 16:04:04 -0800 Subject: [PATCH 098/453] Bump version to 0.4.1-alpha --- multipart/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index b1c138a7a..fb921e09e 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "multipart" -version = "0.4.0" +version = "0.4.1-alpha" authors = ["Austin Bonander "] description = "A backend-agnostic extension for HTTP libraries that provides support for POST multipart/form-data requests on for both client and server." From d5290a737be6cbecffd60ff6498fdc8356c572a1 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Thu, 11 Feb 2016 21:12:07 -0800 Subject: [PATCH 099/453] Bump version to 0.4.1 --- multipart/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index fb921e09e..4e1390145 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "multipart" -version = "0.4.1-alpha" +version = "0.4.1" authors = ["Austin Bonander "] description = "A backend-agnostic extension for HTTP libraries that provides support for POST multipart/form-data requests on for both client and server." From a86caa9921f45352894525f66c0d7379b8c43e4c Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sat, 12 Mar 2016 21:50:42 -0800 Subject: [PATCH 100/453] Change license to Apache-2.0/MIT --- multipart/Cargo.toml | 2 +- multipart/LICENSE-APACHE | 201 +++++++++++++++++++++++++++++++ multipart/LICENSE-MIT | 25 ++++ multipart/src/client/hyper.rs | 6 + multipart/src/client/mod.rs | 6 + multipart/src/client/sized.rs | 6 + multipart/src/lib.rs | 6 + multipart/src/local_test.rs | 6 + multipart/src/server/boundary.rs | 6 + multipart/src/server/hyper.rs | 6 + multipart/src/server/mod.rs | 6 + 11 files changed, 275 insertions(+), 1 deletion(-) create mode 100644 multipart/LICENSE-APACHE create mode 100644 multipart/LICENSE-MIT diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 4e1390145..8b90b4154 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -10,7 +10,7 @@ repository = "http://github.com/cybergeek94/multipart" documentation = "http://rust-ci.org/cybergeek94/multipart/doc/multipart/" -license = "MIT" +license = "MIT OR Apache-2.0" keywords = ["form-data", "hyper", "http", "post", "upload"] diff --git a/multipart/LICENSE-APACHE b/multipart/LICENSE-APACHE new file mode 100644 index 000000000..16fe87b06 --- /dev/null +++ b/multipart/LICENSE-APACHE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/multipart/LICENSE-MIT b/multipart/LICENSE-MIT new file mode 100644 index 000000000..25597d583 --- /dev/null +++ b/multipart/LICENSE-MIT @@ -0,0 +1,25 @@ +Copyright (c) 2010 The Rust Project Developers + +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/multipart/src/client/hyper.rs b/multipart/src/client/hyper.rs index f4ef10017..5f8465f61 100644 --- a/multipart/src/client/hyper.rs +++ b/multipart/src/client/hyper.rs @@ -1,3 +1,9 @@ +// Copyright 2016 `multipart` Crate Developers +// +// Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be +// copied, modified, or distributed except according to those terms. //! Client-side integration with [Hyper](https://github.com/hyperium/hyper). //! Enabled with the `hyper` feature (on by default). //! diff --git a/multipart/src/client/mod.rs b/multipart/src/client/mod.rs index 4ca770ca6..280bea0dd 100644 --- a/multipart/src/client/mod.rs +++ b/multipart/src/client/mod.rs @@ -1,3 +1,9 @@ +// Copyright 2016 `multipart` Crate Developers +// +// Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be +// copied, modified, or distributed except according to those terms. //! The client-side abstraction for multipart requests. Enabled with the `client` feature (on by //! default). //! diff --git a/multipart/src/client/sized.rs b/multipart/src/client/sized.rs index a5dd18145..ed2e7825b 100644 --- a/multipart/src/client/sized.rs +++ b/multipart/src/client/sized.rs @@ -1,3 +1,9 @@ +// Copyright 2016 `multipart` Crate Developers +// +// Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be +// copied, modified, or distributed except according to those terms. //! Sized/buffered wrapper around `HttpRequest`. use client::{HttpRequest, HttpStream}; diff --git a/multipart/src/lib.rs b/multipart/src/lib.rs index 314c7cfbc..0cc982083 100644 --- a/multipart/src/lib.rs +++ b/multipart/src/lib.rs @@ -1,3 +1,9 @@ +// Copyright 2016 `multipart` Crate Developers +// +// Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be +// copied, modified, or distributed except according to those terms. //! Client- and server-side abstractions for HTTP `multipart/form-data` requests. //! //! Features: diff --git a/multipart/src/local_test.rs b/multipart/src/local_test.rs index d082cdc9d..16c96e8fd 100644 --- a/multipart/src/local_test.rs +++ b/multipart/src/local_test.rs @@ -1,3 +1,9 @@ +// Copyright 2016 `multipart` Crate Developers +// +// Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be +// copied, modified, or distributed except according to those terms. use client::HttpRequest as ClientRequest; use client::HttpStream as ClientStream; diff --git a/multipart/src/server/boundary.rs b/multipart/src/server/boundary.rs index f7294988d..03e105f9b 100644 --- a/multipart/src/server/boundary.rs +++ b/multipart/src/server/boundary.rs @@ -1,3 +1,9 @@ +// Copyright 2016 `multipart` Crate Developers +// +// Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be +// copied, modified, or distributed except according to those terms. extern crate buf_redux; extern crate memchr; diff --git a/multipart/src/server/hyper.rs b/multipart/src/server/hyper.rs index 93dbbdc2d..5579cbca5 100644 --- a/multipart/src/server/hyper.rs +++ b/multipart/src/server/hyper.rs @@ -1,3 +1,9 @@ +// Copyright 2016 `multipart` Crate Developers +// +// Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be +// copied, modified, or distributed except according to those terms. //! Server-side integration with [Hyper](https://github.com/hyperium/hyper). //! Enabled with the `hyper` feature (on by default). use hyper::net::Fresh; diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index b5632f513..f0efd1e67 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -1,3 +1,9 @@ +// Copyright 2016 `multipart` Crate Developers +// +// Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be +// copied, modified, or distributed except according to those terms. //! The server-side abstraction for multipart requests. Enabled with the `server` feature (on by //! default). //! From 4fe00a1fbafb4cde871017609a549892cdc57403 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sat, 12 Mar 2016 22:07:03 -0800 Subject: [PATCH 101/453] Add new license info to README --- multipart/README.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/multipart/README.md b/multipart/README.md index 66a4ff16b..da1be8459 100644 --- a/multipart/README.md +++ b/multipart/README.md @@ -75,4 +75,16 @@ fn main() { License ------- -MIT + +Licensed under either of + + * Apache License, Version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or http://www.apache.org/licenses/LICENSE-2.0) + * MIT license ([LICENSE-MIT](LICENSE-MIT) or http://opensource.org/licenses/MIT) + +at your option. + +### Contribution + +Unless you explicitly state otherwise, any contribution intentionally submitted +for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any +additional terms or conditions. From 398db25a812e4154e8d7c3b803a69ddb3b6a378f Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Tue, 1 Mar 2016 14:16:17 -0800 Subject: [PATCH 102/453] Lazy dynamic client --- multipart/Cargo.toml | 2 +- multipart/src/client/hyper.rs | 17 +- multipart/src/client/lazy.rs | 451 ++++++++++++++++++++++++++++++++++ multipart/src/client/mod.rs | 163 +++++++----- 4 files changed, 575 insertions(+), 58 deletions(-) create mode 100644 multipart/src/client/lazy.rs diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 8b90b4154..c44ac9985 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "multipart" -version = "0.4.1" +version = "0.5.0" authors = ["Austin Bonander "] description = "A backend-agnostic extension for HTTP libraries that provides support for POST multipart/form-data requests on for both client and server." diff --git a/multipart/src/client/hyper.rs b/multipart/src/client/hyper.rs index 5f8465f61..2d6e4e34f 100644 --- a/multipart/src/client/hyper.rs +++ b/multipart/src/client/hyper.rs @@ -8,17 +8,25 @@ //! Enabled with the `hyper` feature (on by default). //! //! Contains `impl HttpRequest for Request` and `impl HttpStream for Request`. +//! +//! Also see: [`lazy::Multipart::client_request()`][lazy-multi-creq] +//! and [`lazy::Multipart::client_request_mut()`][lazy-multi-creq-mut] +//! (adaptors for `hyper::client::RequestBuilder`). +//! [lazy-multipart-creq]: ../lazy/struct.Multipart.html#method.client_request +//! [lazy-mutlipart-creq-mut]: ../lazy/struct.Multipart.html#method.client_request_mut use hyper::client::request::Request; use hyper::client::response::Response; -use hyper::error::Error as HyperError; use hyper::header::{ContentType, ContentLength}; use hyper::method::Method; use hyper::net::{Fresh, Streaming}; +use hyper::Error as HyperError; + use mime::{Mime, TopLevel, SubLevel, Attr, Value}; use super::{HttpRequest, HttpStream}; +/// ####Feature: `hyper` impl HttpRequest for Request { type Stream = Request; type Error = HyperError; @@ -53,6 +61,7 @@ impl HttpRequest for Request { } } +/// ####Feature: `hyper` impl HttpStream for Request { type Request = Request; type Response = Response; @@ -63,10 +72,14 @@ impl HttpStream for Request { } } +/// Create a `Content-Type: multipart/form-data;boundary={bound}` +pub fn content_type(bound: &str) -> ContentType { + ContentType(multipart_mime(bound)) +} + fn multipart_mime(bound: &str) -> Mime { Mime( TopLevel::Multipart, SubLevel::Ext("form-data".into()), vec![(Attr::Ext("boundary".into()), Value::Ext(bound.into()))] ) } - diff --git a/multipart/src/client/lazy.rs b/multipart/src/client/lazy.rs new file mode 100644 index 000000000..9521b5088 --- /dev/null +++ b/multipart/src/client/lazy.rs @@ -0,0 +1,451 @@ +//! Multipart requests which write out their data in one fell swoop. + +use mime::Mime; + +use std::borrow::Cow; +use std::error::Error; +use std::fs::File; +use std::path::Path; + +use std::io::prelude::*; +use std::{fmt, io, mem}; + +use super::{HttpRequest, HttpStream, MultipartWriter}; + +macro_rules! try_lazy ( + ($field:expr, $try:expr) => ( + match $try { + Ok(ok) => ok, + Err(e) => return Err(LazyError::with_field($field, e)), + } + ); + ($try:expr) => ( + match $try { + Ok(ok) => ok, + Err(e) => return Err(LazyError::without_field(e)), + } + ) +); + +/// A `LazyError` wrapping `std::io::Error`. +pub type LazyIoError<'a> = LazyError<'a, io::Error>; + +/// An error for lazily written multipart requests, including the original error as well +/// as the field which caused the error, if applicable. +pub struct LazyError<'a, E> { + /// The field that caused the error. + /// If `None`, there was a problem opening the stream to write or finalizing the stream. + pub field_name: Option>, + /// The inner error. + pub error: E, + /// Private field for back-compat. + _priv: (), +} + +impl<'a, E> LazyError<'a, E> { + fn without_field>(error: E_) -> Self { + LazyError { + field_name: None, + error: error.into(), + _priv: (), + } + } + + fn with_field>(field_name: Cow<'a, str>, error: E_) -> Self { + LazyError { + field_name: Some(field_name), + error: error.into(), + _priv: (), + } + } +} + +/// Take `self.error`, discarding `self.field_name`. +impl<'a> Into for LazyError<'a, io::Error> { + fn into(self) -> io::Error { + self.error + } +} + +impl<'a, E: Error> Error for LazyError<'a, E> { + fn description(&self) -> &str { + self.error.description() + } + + fn cause(&self) -> Option<&Error> { + Some(&self.error) + } +} + +impl<'a, E: fmt::Debug> fmt::Debug for LazyError<'a, E> { + fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + if let Some(ref field_name) = self.field_name { + fmt.write_fmt(format_args!("LazyError (on field {:?}): {:?}", field_name, self.error)) + } else { + fmt.write_fmt(format_args!("LazyError (misc): {:?}", self.error)) + } + } +} + +impl<'a, E: fmt::Display> fmt::Display for LazyError<'a, E> { + fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + if let Some(ref field_name) = self.field_name { + fmt.write_fmt(format_args!("Error writing field {:?}: {}", field_name, self.error)) + } else { + fmt.write_fmt(format_args!("Error opening or flushing stream: {}", self.error)) + } + } +} + +/// A multipart request which writes all fields at once upon being provided an output stream. +/// +/// Sacrifices static dispatch for support for dynamic construction. Reusable. +/// +/// ####Lifetimes +/// * `'n`: Lifetime for field **n**ames; will only escape this struct in `LazyIoError<'n>`. +/// * `'d`: Lifetime for **d**ata: will only escape this struct in `PreparedFields<'d>`. +#[derive(Debug, Default)] +pub struct Multipart<'n, 'd> { + fields: Vec>, +} + +impl<'n, 'd> Multipart<'n, 'd> { + /// Initialize a new lazy dynamic request. + pub fn new() -> Self { + Default::default() + } + + /// Add a text field to this request. + pub fn add_text(&mut self, name: N, text: T) -> &mut Self where N: Into>, T: Into> { + self.fields.push( + Field { + name: name.into(), + data: Data::Text(text.into()) + } + ); + + self + } + + /// Add a file field to this request. + /// + /// ### Note + /// Does not check if `path` exists. + pub fn add_file(&mut self, name: N, path: P) -> &mut Self where N: Into>, P: Into> { + self.fields.push( + Field { + name: name.into(), + data: Data::File(path.into()), + } + ); + + self + } + + /// Add a generic stream field to this request, + pub fn add_stream(&mut self, name: N, stream: R, filename: Option, mime: Option) -> &mut Self where N: Into>, R: Read + 'd, F: Into> { + self.fields.push( + Field { + name: name.into(), + data: Data::Stream(Stream { + content_type: mime, + filename: filename.map(|f| f.into()), + stream: Box::new(stream) + }), + } + ); + + self + } + + /// Convert `req` to `HttpStream`, write out the fields in this request, and finish the + /// request, returning the response if successful, or the first error encountered. + pub fn send(&mut self, req: R) -> Result<::Response, LazyError<'n, ::Error>> { + let (boundary, stream) = try_lazy!(super::open_stream(req, None)); + let mut writer = MultipartWriter::new(stream, boundary); + + for mut field in self.fields.drain(..) { + try_lazy!(field.name, field.write_out(&mut writer)); + } + + try_lazy!(writer.finish()).finish().map_err(LazyError::without_field) + } + + /// Export the multipart data contained in this lazy request as an adaptor which implements `Read`. + /// + /// A certain amount of field data will be buffered. See + /// [`prepare_threshold()`](#method.prepare_threshold) for more information on this behavior. + pub fn prepare(&mut self) -> Result, LazyIoError<'n>> { + self.prepare_threshold(Some(DEFAULT_BUFFER_THRESHOLD)) + } + + /// Export the multipart data contained in this lazy request to an adaptor which implements `Read`. + /// + /// #### Buffering + /// For efficiency, text and file fields smaller than `buffer_threshold` are copied to an in-memory buffer. If `None`, + /// all fields are copied to memory. + /// + /// + pub fn prepare_threshold(&mut self, buffer_threshold: Option) -> Result, LazyIoError<'n>> { + let boundary = super::gen_boundary(); + PreparedFields::from_fields(&mut self.fields, boundary.into(), buffer_threshold) + } +} + +const DEFAULT_BUFFER_THRESHOLD: u64 = 8 * 1024; + +#[derive(Debug)] +struct Field<'n, 'd> { + name: Cow<'n, str>, + data: Data<'n, 'd>, +} + +impl<'n, 'd> Field<'n, 'd> { + fn write_out(&mut self, writer: &mut MultipartWriter) -> io::Result<()> { + match self.data { + Data::Text(ref text) => writer.write_text(&self.name, text), + Data::File(ref path) => writer.write_file(&self.name, path), + Data::Stream(ref mut stream) => + writer.write_stream( + &mut stream.stream, + &self.name, + stream.filename.as_ref().map(|f| &**f), + stream.content_type.clone(), + ), + } + } +} + +enum Data<'n, 'd> { + Text(Cow<'d, str>), + File(Cow<'d, Path>), + Stream(Stream<'n, 'd>), +} + +impl<'n, 'd> fmt::Debug for Data<'n, 'd> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match *self { + Data::Text(ref text) => write!(f, "Data::Text({:?})", text), + Data::File(ref path) => write!(f, "Data::File({:?})", path), + Data::Stream(_) => f.write_str("Data::Stream(Box"), + } + } +} + +struct Stream<'n, 'd> { + filename: Option>, + content_type: Option, + stream: Box, +} + +/// The result of [`Multipart::prepare()`](../Multipart.html#method.prepare) or +/// `Multipart::prepare_threshold()`. Implements `Read`, contains the entire request body. +pub struct PreparedFields<'d> { + // NOTE: the order of these fields have been reversed so fields can be popped one-by-one from + // the end. + fields: Vec>, + boundary: String, + content_len: Option, +} + +impl<'d> PreparedFields<'d> { + fn from_fields<'n>(fields: &mut Vec>, boundary: String, buffer_threshold: Option) -> Result> { + let buffer_threshold = buffer_threshold.unwrap_or(u64::max_value()); + + let mut prep_fields = Vec::with_capacity(fields.len()); + + // We reverse so we can pop efficiently from the end + let mut fields = fields.drain(..).rev().peekable(); + + let mut contiguous = Vec::new(); + let mut remainder: Option> = None; + let mut use_content_len = true; + let mut content_len = 0; + + while fields.peek().is_some() { + { + let mut writer = MultipartWriter::new(&mut contiguous, &*boundary); + + while let Some(field) = fields.next() { + match field.data { + Data::Text(text) => if text.len() as u64 <= buffer_threshold { + try_lazy!(field.name, writer.write_text(&field.name, &*text)); + } else { + try_lazy!(field.name, writer.write_field_headers(&field.name, None, None)); + content_len += text.len() as u64; + remainder = Some(Box::new(io::Cursor::new(CowStrAsRef(text)))); + }, + Data::File(path) => { + let (content_type, filename) = super::mime_filename(&*path); + let mut file = try_lazy!(field.name, File::open(&*path)); + let len = try_lazy!(field.name, file.metadata()).len(); + + if len <= buffer_threshold { + try_lazy!(field.name, writer.write_stream(&mut file, &field.name, filename, Some(content_type))); + } else { + try_lazy!(field.name, writer.write_field_headers(&field.name, filename, Some(content_type))); + remainder = Some(Box::new(file)); + content_len += len; + } + }, + Data::Stream(stream) => { + let filename = stream.filename.as_ref().map(|f| &**f); + try_lazy!(field.name, writer.write_field_headers(&field.name, filename, stream.content_type)); + remainder = Some(stream.stream); + use_content_len = false; + }, + } + + if remainder.is_some() { break; } + } + } + + content_len += contiguous.len() as u64; + + let contiguous = io::Cursor::new(mem::replace(&mut contiguous, Vec::new())); + + if let Some(rem) = remainder.take() { + prep_fields.push(PreparedField::Partial(contiguous.chain(rem))); + } else { + prep_fields.push(PreparedField::Contiguous(contiguous)); + } + } + + // FIXME: when non-lexical borrow scopes land, convert this to a single if-let/else + let mut end_written = false; + + if let Some(&mut PreparedField::Contiguous(ref mut vec)) = prep_fields.last_mut() { + let start_len = vec.get_ref().len(); + try_lazy!(write!(vec, "\r\n--{}--", boundary)); + content_len += (vec.get_ref().len() - start_len) as u64; + end_written = true; + } + + if !end_written { + let vec = format!("\r\n--{}--", boundary).into_bytes(); + content_len += vec.len() as u64; + prep_fields.push(PreparedField::Contiguous(io::Cursor::new(vec))); + } + + Ok(PreparedFields { + fields: prep_fields, + boundary: boundary, + content_len: if use_content_len { Some(content_len) } else { None }, + }) + } + + /// Get the content-length value for this set of fields, if applicable (all fields are sized, + /// i.e. not generic streams). + pub fn content_len(&self) -> Option { + self.content_len + } + + fn boundary(&self) -> &str { + &self.boundary + } +} + +impl<'d> Read for PreparedFields<'d> { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + if buf.is_empty() { return Ok(0) } + if self.fields.is_empty() { return Ok(0) } + + let bytes_read = if let Some(mut curr) = self.fields.last_mut() { + try!(curr.read(buf)) + } else { + 0 + }; + + if bytes_read == 0 { + let _ = self.fields.pop(); + } + + Ok(bytes_read) + } +} + +#[doc(hidden)] +pub enum PreparedField<'d> { + Contiguous(io::Cursor>), + Partial(io::Chain>, Box>), +} + +impl<'d> Read for PreparedField<'d> { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + match *self { + PreparedField::Contiguous(ref mut vec) => vec.read(buf), + PreparedField::Partial(ref mut chain) => chain.read(buf), + } + } +} + +struct CowStrAsRef<'d>(Cow<'d, str>); + +impl<'d> AsRef<[u8]> for CowStrAsRef<'d> { + fn as_ref(&self) -> &[u8] { + self.0.as_bytes() + } +} + +#[cfg(feature = "hyper")] +mod hyper { + use hyper::client::{Body, Client, IntoUrl, RequestBuilder, Response}; + use hyper::Result as HyperResult; + + impl<'n, 'd> super::Multipart<'n, 'd> { + /// #### Feature: `hyper` + /// Complete a POST request with the given `hyper::client::Client` and URL. + /// + /// Supplies the fields in the body, optionally setting the content-length header if + /// applicable (all added fields were text or files, i.e. no streams). + pub fn client_request(&mut self, client: &Client, url: U) -> HyperResult { + self.client_request_mut(client, url, |r| r) + } + + /// #### Feature: `hyper` + /// Complete a POST request with the given `hyper::client::Client` and URL; + /// allows mutating the `hyper::client::RequestBuilder` via the passed closure. + /// + /// Note that the body, and the `ContentType` and `ContentLength` headers will be + /// overwritten, either by this method or by Hyper. + pub fn client_request_mut RequestBuilder>(&mut self, client: &Client, url: U, + mut_fn: F) -> HyperResult { + + let mut fields = match self.prepare() { + Ok(fields) => fields, + Err(err) => { + error!("Error preparing request: {}", err); + return Err(err.error.into()); + }, + }; + + + mut_fn(client.post(url)) + .header(::client::hyper::content_type(fields.boundary())) + .body(fields.to_body()) + .send() + } + } + + impl<'d> super::PreparedFields<'d> { + /// #### Feature: `hyper` + /// Convert `self` to `hyper::client::Body`. + pub fn to_body<'b>(&'b mut self) -> Body<'b> { + use super::PreparedField; + // We have a single contiguous body, provide it directly + if self.fields.len() == 1 { + if let PreparedField::Contiguous(ref body) = self.fields[0] { + return Body::BufBody(body.get_ref(), body.get_ref().len()); + } else { + unreachable!("Only one field but it was not contiguous!"); + } + } + + if let Some(content_len) = self.content_len { + Body::SizedBody(self, content_len) + } else { + Body::ChunkedBody(self) + } + } + } +} diff --git a/multipart/src/client/mod.rs b/multipart/src/client/mod.rs index 280bea0dd..73ffd651f 100644 --- a/multipart/src/client/mod.rs +++ b/multipart/src/client/mod.rs @@ -10,6 +10,7 @@ //! Use this when sending POST requests with files to a server. use mime::Mime; +use std::borrow::Cow; use std::fs::File; use std::io; use std::io::prelude::*; @@ -19,23 +20,22 @@ use std::path::Path; #[cfg(feature = "hyper")] pub mod hyper; +pub mod lazy; + mod sized; pub use self::sized::SizedRequest; const BOUNDARY_LEN: usize = 16; - /// The entry point of the client-side multipart API. /// /// Though they perform I/O, the `.write_*()` methods do not return `io::Result<_>` in order to /// facilitate method chaining. Upon the first error, all subsequent API calls will be no-ops until /// `.send()` is called, at which point the error will be reported. pub struct Multipart { - stream: S, - boundary: String, + writer: MultipartWriter<'static, S>, last_err: Option, - data_written: bool, } impl Multipart { @@ -43,17 +43,12 @@ impl Multipart { /// /// ## Returns Error /// If `req.open_stream()` returns an error. - pub fn from_request(mut req: R) -> Result, R::Error> { - let boundary = ::random_alphanumeric(BOUNDARY_LEN); - req.apply_headers(&boundary, None); - - let stream = try!(req.open_stream()); + pub fn from_request(req: R) -> Result, R::Error> { + let (boundary, stream) = try!(open_stream(req, None)); Ok(Multipart { - stream: stream, - boundary: boundary, + writer: MultipartWriter::new(stream, boundary), last_err: None, - data_written: false, }) } } @@ -80,10 +75,8 @@ impl Multipart { /// If something went wrong with the HTTP stream. pub fn write_text, V: AsRef>(&mut self, name: N, val: V) -> &mut Self { if self.last_err.is_none() { - self.last_err = chain_result! { - self.write_field_headers(name.as_ref(), None, None), - self.stream.write_all(val.as_ref().as_bytes()) - }.err().map(|err| err.into()) + self.last_err = self.writer.write_text(name.as_ref(), val.as_ref()) + .err().map(|err| err.into()) } self @@ -101,17 +94,12 @@ impl Multipart { /// If there was a problem opening the file (was a directory or didn't exist), /// or if something went wrong with the HTTP stream. pub fn write_file, P: AsRef>(&mut self, name: N, path: P) -> &mut Self { - if self.last_err.is_none() { + if self.last_err.is_none() { + let name = name.as_ref(); let path = path.as_ref(); - self.last_err = chain_result! { - { // New borrow scope so we can reborrow `file` after - let content_type = ::mime_guess::guess_mime_type(path); - let filename = path.file_name().and_then(|filename| filename.to_str()); - self.write_field_headers(name.as_ref(), filename, Some(content_type)) - }, - File::open(path).and_then(|ref mut file| io::copy(file, &mut self.stream)) - }.err().map(|err| err.into()); + self.last_err = self.writer.write_file(name, path).err() + .map(|err| err.into()); } self @@ -138,46 +126,23 @@ impl Multipart { /// If the reader returned an error, or if something went wrong with the HTTP stream. // RFC: How to format this declaration? pub fn write_stream, St: Read>( - &mut self, name: N, read: &mut St, filename: Option<&str>, content_type: Option + &mut self, name: N, stream: &mut St, filename: Option<&str>, content_type: Option ) -> &mut Self { if self.last_err.is_none() { - let content_type = content_type.unwrap_or_else(::mime_guess::octet_stream); + let name = name.as_ref(); - self.last_err = chain_result! { - self.write_field_headers(name.as_ref(), filename, Some(content_type)), - io::copy(read, &mut self.stream) - }.err().map(|err| err.into()); + self.last_err = self.writer.write_stream(stream, name, filename, content_type) + .err().map(|err| err.into()); } self } - fn write_field_headers(&mut self, name: &str, filename: Option<&str>, content_type: Option) - -> io::Result<()> { - self.data_written = true; - - chain_result! { - // Write the first boundary, or the boundary for the previous field. - write!(self.stream, "\r\n--{}\r\n", self.boundary), - write!(self.stream, "Content-Disposition: form-data; name=\"{}\"", name), - filename.map(|filename| write!(self.stream, "; filename=\"{}\"", filename)) - .unwrap_or(Ok(())), - content_type.map(|content_type| write!(self.stream, "\r\nContent-Type: {}", content_type)) - .unwrap_or(Ok(())), - self.stream.write_all(b"\r\n\r\n") - } - } - /// Finalize the request and return the response from the server, or the last error if set. - pub fn send(mut self) -> Result { + pub fn send(self) -> Result { match self.last_err { - None => { - if self.data_written { - // Write two hyphens after the last boundary occurrence. - try!(write!(self.stream, "\r\n--{}--", self.boundary)); - } - - self.stream.finish() + None => { + try!(self.writer.finish()).finish() }, Some(err) => Err(err), } @@ -241,3 +206,91 @@ impl HttpStream for io::Sink { fn finish(self) -> Result { Ok(()) } } + +fn gen_boundary() -> String { + ::random_alphanumeric(BOUNDARY_LEN) +} + +fn open_stream(mut req: R, content_len: Option) -> Result<(String, R::Stream), R::Error> { + let boundary = gen_boundary(); + req.apply_headers(&boundary, content_len); + req.open_stream().map(|stream| (boundary, stream)) +} + +struct MultipartWriter<'a, W> { + inner: W, + boundary: Cow<'a, str>, + data_written: bool, +} + +impl<'a, W: Write> MultipartWriter<'a, W> { + fn new>>(inner: W, boundary: B) -> Self { + MultipartWriter { + inner: inner, + boundary: boundary.into(), + data_written: false, + } + } + + fn write_boundary(&mut self) -> io::Result<()> { + write!(self.inner, "\r\n--{}\r\n", self.boundary) + } + + fn write_text(&mut self, name: &str, text: &str) -> io::Result<()> { + chain_result! { + self.write_field_headers(name, None, None), + self.inner.write_all(text.as_bytes()) + } + } + + fn write_file(&mut self, name: &str, path: &Path) -> io::Result<()> { + let (content_type, filename) = mime_filename(path); + let mut file = try!(File::open(path)); + self.write_stream(&mut file, name, filename, Some(content_type)) + } + + fn write_stream(&mut self, stream: &mut S, name: &str, filename: Option<&str>, content_type: Option) -> io::Result<()> { + // This is necessary to make sure it is interpreted as a file on the server end. + let content_type = Some(content_type.unwrap_or_else(::mime_guess::octet_stream)); + + chain_result! { + self.write_field_headers(name, filename, content_type), + io::copy(stream, &mut self.inner), + Ok(()) + } + } + + fn write_field_headers(&mut self, name: &str, filename: Option<&str>, content_type: Option) + -> io::Result<()> { + chain_result! { + // Write the first boundary, or the boundary for the previous field. + self.write_boundary(), + { self.data_written = true; Ok(()) }, + write!(self.inner, "Content-Disposition: form-data; name=\"{}\"", name), + filename.map(|filename| write!(self.inner, "; filename=\"{}\"", filename)) + .unwrap_or(Ok(())), + content_type.map(|content_type| write!(self.inner, "\r\nContent-Type: {}", content_type)) + .unwrap_or(Ok(())), + self.inner.write_all(b"\r\n\r\n") + } + } + + fn finish(mut self) -> io::Result { + if self.data_written { + // Write two hyphens after the last boundary occurrence. + try!(write!(self.inner, "\r\n--{}--", self.boundary)); + } + + Ok(self.inner) + } +} + +fn mime_filename(path: &Path) -> (Mime, Option<&str>) { + let content_type = ::mime_guess::guess_mime_type(path); + let filename = opt_filename(path); + (content_type, filename) +} + +fn opt_filename(path: &Path) -> Option<&str> { + path.file_name().and_then(|filename| filename.to_str()) +} From 3da866563dc0de1755f3e0ea2c3ec7d39a4a5433 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Mon, 7 Mar 2016 00:07:33 -0800 Subject: [PATCH 103/453] More flexible HttpRequest --- multipart/src/server/boundary.rs | 2 +- multipart/src/server/hyper.rs | 6 +++ multipart/src/server/mod.rs | 82 ++++++++++++++++++++------------ 3 files changed, 58 insertions(+), 32 deletions(-) diff --git a/multipart/src/server/boundary.rs b/multipart/src/server/boundary.rs index 03e105f9b..1e1f4235d 100644 --- a/multipart/src/server/boundary.rs +++ b/multipart/src/server/boundary.rs @@ -136,7 +136,7 @@ impl BoundaryReader where R: Read { } } -impl Borrow for BoundaryReader { +impl Borrow for BoundaryReader { fn borrow(&self) -> &R { self.buf.get_ref() } diff --git a/multipart/src/server/hyper.rs b/multipart/src/server/hyper.rs index 5579cbca5..cc690eaa9 100644 --- a/multipart/src/server/hyper.rs +++ b/multipart/src/server/hyper.rs @@ -68,6 +68,8 @@ where F: Fn(Multipart, Response), F: Send + Sync { } impl<'a, 'b> HttpRequest for Request<'a, 'b> { + type Body = Self; + fn multipart_boundary(&self) -> Option<&str> { if self.method != Method::Post { return None; @@ -93,5 +95,9 @@ impl<'a, 'b> HttpRequest for Request<'a, 'b> { ) }) } + + fn body(&mut self) -> &mut Self { + self + } } diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index f0efd1e67..760f65ec6 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -42,8 +42,8 @@ macro_rules! try_opt ( /// The server-side implementation of `multipart/form-data` requests. /// /// Implements `Borrow` to allow access to the request object. -pub struct Multipart { - source: BoundaryReader, +pub struct Multipart { + source: BoundaryReader, line_buf: String, /// The directory for saving files in this request. /// By default, this is set to a subdirectory of `std::env::temp_dir()` with a @@ -51,25 +51,40 @@ pub struct Multipart { pub save_dir: PathBuf, } -impl Multipart where R: HttpRequest { - /// If the given `R: HttpRequest` is a POST request of `Content-Type: multipart/form-data`, - /// return the wrapped request as `Ok(Multipart)`, otherwise `Err(R)`. - pub fn from_request(req: R) -> Result, R> { - if req.multipart_boundary().is_none() { - return Err(req); - } - let boundary = format!("--{}", req.multipart_boundary().unwrap()); +impl Multipart<()> { + /// If the given `R: HttpRequest` is a POST request of `Content-Type: multipart/form-data`, + /// return the wrapped request as `Ok(Multipart)`, otherwise `Err(R)`. + pub fn from_request(req: R) -> Result, R> { + //FIXME: move `map` expr to `Some` arm when nonlexical borrow scopes land. + let boundary = match req.multipart_boundary().map(|b| format!("--{}", b)){ + Some(boundary) => boundary, + None => return Err(req), + }; debug!("Boundary: {}", boundary); Ok( Multipart { - source: BoundaryReader::from_reader(req, boundary), + source: BoundaryReader::from_reader(req.body(), boundary), line_buf: String::new(), save_dir: ::temp_dir(), } - ) + ) + } +} + +impl Multipart { + pub fn with_body(body: B, boundary: &str) -> Self { + let boundary = format!("--{}", boundary); + + debug!("Boundary: {}", boundary); + + Multipart { + source: BoundaryReader::from_reader(body, boundary), + line_buf: String::new(), + save_dir: ::temp_dir(), + } } /// Read the next entry from this multipart request, returning a struct with the field's name and @@ -78,7 +93,7 @@ impl Multipart where R: HttpRequest { /// ##Warning: Risk of Data Loss /// If the previously returned entry had contents of type `MultipartField::File`, /// calling this again will discard any unread contents of that entry. - pub fn read_entry(&mut self) -> io::Result>> { + pub fn read_entry(&mut self) -> io::Result>> { if !try!(self.consume_boundary()) { return Ok(None); } @@ -97,7 +112,7 @@ impl Multipart where R: HttpRequest { /// from `next()` borrows the iterator for a bound lifetime). /// /// Returns `Ok(())` when all fields have been read, or the first error. - pub fn foreach_entry(&mut self, mut foreach: F) -> io::Result<()> where F: FnMut(MultipartField) { + pub fn foreach_entry(&mut self, mut foreach: F) -> io::Result<()> where F: FnMut(MultipartField) { loop { match self.read_entry() { Ok(Some(field)) => foreach(field), @@ -174,8 +189,8 @@ impl Multipart where R: HttpRequest { } } -impl Borrow for Multipart where R: HttpRequest { - fn borrow(&self) -> &R { +impl Borrow for Multipart { + fn borrow(&self) -> &B { self.source.borrow() } } @@ -267,26 +282,31 @@ fn get_remainder_after<'a>(needle: &str, haystack: &'a str) -> Option<(&'a str)> } /// A server-side HTTP request that may or may not be multipart. -pub trait HttpRequest: Read { +pub trait HttpRequest { + /// The body of this request. + type Body: Read; /// Get the boundary string of this request if it is a POST request /// with the `Content-Type` header set to `multipart/form-data`. /// /// The boundary string should be supplied as an extra value of the `Content-Type` header, e.g. /// `Content-Type: multipart/form-data; boundary=myboundary`. fn multipart_boundary(&self) -> Option<&str>; + + /// Return a mutable reference to the request body for reading. + fn body(self) -> Self::Body; } /// A field in a multipart request. May be either text or a binary stream (file). #[derive(Debug)] -pub struct MultipartField<'a, R: 'a> { +pub struct MultipartField<'a, B: 'a> { /// The field's name from the form pub name: String, /// The data of the field. Can be text or binary. - pub data: MultipartData<'a, R>, + pub data: MultipartData<'a, B>, } -impl<'a, R: HttpRequest + 'a> MultipartField<'a, R> { - fn read_from(multipart: &'a mut Multipart) -> io::Result>> { +impl<'a, B: Read + 'a> MultipartField<'a, B> { + fn read_from(multipart: &'a mut Multipart) -> io::Result>> { let cont_disp = match multipart.read_content_disposition() { Ok(Some(cont_disp)) => cont_disp, Ok(None) => return Ok(None), @@ -326,16 +346,16 @@ impl<'a, R: HttpRequest + 'a> MultipartField<'a, R> { /// The data of a field in a `multipart/form-data` request. #[derive(Debug)] -pub enum MultipartData<'a, R: 'a> { +pub enum MultipartData<'a, B: 'a> { /// The field's payload is a text string. Text(&'a str), /// The field's payload is a binary stream (file). - File(MultipartFile<'a, R>), + File(MultipartFile<'a, B>), // TODO: Support multiple files per field (nested boundaries) // MultiFiles(Vec), } -impl<'a, R> MultipartData<'a, R> { +impl<'a, B> MultipartData<'a, B> { /// Borrow this payload as a text field, if possible. pub fn as_text(&self) -> Option<&str> { match *self { @@ -346,7 +366,7 @@ impl<'a, R> MultipartData<'a, R> { /// Borrow this payload as a file field, if possible. /// Mutably borrows so the contents can be read. - pub fn as_file(&mut self) -> Option<&mut MultipartFile<'a, R>> { + pub fn as_file(&mut self) -> Option<&mut MultipartFile<'a, B>> { match *self { MultipartData::File(ref mut file) => Some(file), _ => None, @@ -363,18 +383,18 @@ impl<'a, R> MultipartData<'a, R> { /// You can read it to EOF, or use one of the `save_*()` methods here /// to save it to disk. #[derive(Debug)] -pub struct MultipartFile<'a, R: 'a> { +pub struct MultipartFile<'a, B: 'a> { filename: Option, content_type: Mime, save_dir: &'a Path, - stream: &'a mut BoundaryReader, + stream: &'a mut BoundaryReader, } -impl<'a, R: Read> MultipartFile<'a, R> { +impl<'a, B: Read> MultipartFile<'a, B> { fn from_stream(filename: Option, content_type: Mime, save_dir: &'a Path, - stream: &'a mut BoundaryReader) -> MultipartFile<'a, R> { + stream: &'a mut BoundaryReader) -> MultipartFile<'a, B> { MultipartFile { filename: filename, content_type: content_type, @@ -457,13 +477,13 @@ impl<'a, R: Read> MultipartFile<'a, R> { } } -impl<'a, R: Read> Read for MultipartFile<'a, R> { +impl<'a, B: Read> Read for MultipartFile<'a, B> { fn read(&mut self, buf: &mut [u8]) -> io::Result{ self.stream.read(buf) } } -impl<'a, R: Read> BufRead for MultipartFile<'a, R> { +impl<'a, B: Read> BufRead for MultipartFile<'a, B> { fn fill_buf(&mut self) -> io::Result<&[u8]> { self.stream.fill_buf() } From a72f23e5d650eb21b46bf08abefc1f54fb1996d8 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Mon, 7 Mar 2016 00:15:18 -0800 Subject: [PATCH 104/453] Add impl for &mut hyper::server::Request --- multipart/src/server/hyper.rs | 36 ++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/multipart/src/server/hyper.rs b/multipart/src/server/hyper.rs index cc690eaa9..3186d6801 100644 --- a/multipart/src/server/hyper.rs +++ b/multipart/src/server/hyper.rs @@ -96,7 +96,41 @@ impl<'a, 'b> HttpRequest for Request<'a, 'b> { }) } - fn body(&mut self) -> &mut Self { + fn body(self) -> Self { + self + } +} + +impl<'r, 'a, 'b> HttpRequest for &'r mut Request<'a, 'b> { + type Body = &'r mut Request<'a, 'b>; + + fn multipart_boundary(&self) -> Option<&str> { + if self.method != Method::Post { + return None; + } + + self.headers.get::().and_then(|ct| { + let ContentType(ref mime) = *ct; + let params = match *mime { + Mime(TopLevel::Multipart, SubLevel::FormData, ref params) => params, + _ => return None, + }; + + params.iter().find(|&&(ref name, _)| + match *name { + Attr::Boundary => true, + _ => false, + } + ).and_then(|&(_, ref val)| + match *val { + Value::Ext(ref val) => Some(&**val), + _ => None, + } + ) + }) + } + + fn body(self) -> Self::Body { self } } From 8570a82d44cf7af6d3e57985a86660d40b1f8338 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Mon, 7 Mar 2016 00:11:08 -0800 Subject: [PATCH 105/453] Bootstrap Iron, tiny_http --- multipart/Cargo.toml | 6 ++- multipart/src/lib.rs | 6 +++ multipart/src/server/iron.rs | 86 +++++++++++++++++++++++++++++++ multipart/src/server/tiny_http.rs | 13 +++++ 4 files changed, 110 insertions(+), 1 deletion(-) create mode 100644 multipart/src/server/iron.rs create mode 100644 multipart/src/server/tiny_http.rs diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index c44ac9985..ae715268e 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -12,7 +12,7 @@ documentation = "http://rust-ci.org/cybergeek94/multipart/doc/multipart/" license = "MIT OR Apache-2.0" -keywords = ["form-data", "hyper", "http", "post", "upload"] +keywords = ["form-data", "hyper", "iron", "http", "post", "upload"] [features] server = ["buf_redux", "memchr"] @@ -30,6 +30,10 @@ rand = "0.3" version = "0.7" optional = true +[dependencies.iron] +version = "0.2" +optional = true + [dependencies.buf_redux] version = "0.1" optional = true diff --git a/multipart/src/lib.rs b/multipart/src/lib.rs index 0cc982083..ccb876147 100644 --- a/multipart/src/lib.rs +++ b/multipart/src/lib.rs @@ -27,6 +27,12 @@ extern crate rand; #[cfg(feature = "hyper")] extern crate hyper; +#[cfg(feature = "iron")] +extern crate iron; + +#[cfg(feature = "tiny_http")] +extern crate tiny_http; + use rand::Rng; use std::path::PathBuf; diff --git a/multipart/src/server/iron.rs b/multipart/src/server/iron.rs new file mode 100644 index 000000000..75a3af6b1 --- /dev/null +++ b/multipart/src/server/iron.rs @@ -0,0 +1,86 @@ +//! Adaptor types and impls for `iron::Request`. Includes a `BeforeMiddleware` implementation. + +use iron::headers::ContentType; +use iron::BeforeMiddleware; +use iron::mime::{Mime, TopLevel, SubLevel}; +use iron::request::{Body, Request}; + +use tempdir::TempDir; + +use std::collections::HashMap; +use std::path::PathBuf; + +use super::{HttpRequest, Multipart}; + +impl<'r, 'a, 'b> HttpRequest for &'r mut Request<'a, 'b> { + type Body = &'r mut Body<'a, 'b>; + + fn multipart_boundary(&self) -> Option<&str> { + let content_type = try_opt!(self.headers.get::()); + if let Mime(TopLevel::Multipart, SubLevel::FormData, _) = *content_type { + content_type.get_param("boundary").map(|b| b.as_str()) + } else { + None + } + } + + fn body(self) -> &'r mut Body<'a, 'b> { + &mut self.body + } +} + +/// The default file size limit for `Intercept`, in bytes. +pub const DEFAULT_FILE_SIZE_LIMIT: u64 = 2 * 1024 * 1024; + +/// The default file count limit for `Intercept`. +pub const DEFAULT_FILE_COUNT_LIMIT: u64 = 16; + +pub struct Intercept { + pub temp_dir_path: Option, + pub file_size_limit: u64, + pub file_count_limit: u64, +} + +impl Intercept { + pub fn new() -> Self { + Default::default() + } + + pub fn temp_dir_path>(self, path: P) -> Self { + Intercept { temp_dir_path: path.into(), .. self } + } + + pub fn file_size_limit(self, limit: u64) -> Self { + Intercept { file_size_limit: limit, .. self } + } + + pub fn file_count_limit(self, limit: u64) -> Self { + Intercept { file_count_limit: limit, .. self } + } +} + +impl Default for Intercept { + fn default() -> Self { + Intercept { + temp_dir_path: None, + file_size_limit: DEFAULT_FILE_SIZE_LIMIT, + file_count_limit: DEFAULT_FILE_COUNT_LIMIT, + } + } +} + +impl BeforeMiddleware for Intercept { + fn before(&self, req: &mut Request) -> IronResult<()> { + + } +} + +pub enum LimitBehavior { + ThrowError, + Continue, +} + +struct LimitReader { + inner: R, + limit: u64, +} diff --git a/multipart/src/server/tiny_http.rs b/multipart/src/server/tiny_http.rs new file mode 100644 index 000000000..13c1aa048 --- /dev/null +++ b/multipart/src/server/tiny_http.rs @@ -0,0 +1,13 @@ +use tiny_http::Request; + +use super::HttpRequest; + +use std::io::Read; + +impl<'r> HttpRequest for &'r mut Request { + type Body = &'r mut Read; + + fn multipart_boundary(&self) -> Option<&str> { + self.headers.iter().find(|header| + } +} From 7e0be07099b3348ce7f5ca1907f583ccae6c66c7 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Mon, 7 Mar 2016 00:18:56 -0800 Subject: [PATCH 106/453] Use tempdir for server file saves --- multipart/Cargo.toml | 1 + multipart/src/lib.rs | 12 +- multipart/src/server/mod.rs | 424 +++++++++++++++++++++++++++--------- 3 files changed, 326 insertions(+), 111 deletions(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index ae715268e..c891b69f2 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -25,6 +25,7 @@ env_logger = "0.3" mime = "0.1" mime_guess = "1.4" rand = "0.3" +tempdir = "0.3" [dependencies.hyper] version = "0.7" diff --git a/multipart/src/lib.rs b/multipart/src/lib.rs index ccb876147..deb85ff21 100644 --- a/multipart/src/lib.rs +++ b/multipart/src/lib.rs @@ -24,6 +24,8 @@ extern crate mime; extern crate mime_guess; extern crate rand; +extern crate tempdir; + #[cfg(feature = "hyper")] extern crate hyper; @@ -35,12 +37,10 @@ extern crate tiny_http; use rand::Rng; -use std::path::PathBuf; - macro_rules! chain_result { ($first_expr:expr, $($try_expr:expr),*) => ( $first_expr $(.and_then(|_| $try_expr))* - ); + ); } #[cfg(feature = "client")] @@ -51,12 +51,6 @@ pub mod server; #[cfg(all(test, feature = "client", feature = "server"))] mod local_test; -const DIRNAME_LEN: usize = 12; - -fn temp_dir() -> PathBuf { - random_alphanumeric(DIRNAME_LEN).into() -} - fn random_alphanumeric(len: usize) -> String { rand::thread_rng().gen_ascii_chars().take(len).collect() } diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index 760f65ec6..d0274137c 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -13,23 +13,17 @@ //! See the `Multipart` struct for more info. use mime::Mime; -use std::borrow::Borrow; +use tempdir::TempDir; +use std::borrow::Borrow; use std::collections::HashMap; - use std::fs::{self, File}; -use std::io; use std::io::prelude::*; - use std::path::{Path, PathBuf}; +use std::{fmt, io, mem, ptr}; pub use self::boundary::BoundaryReader; -mod boundary; - -#[cfg(feature = "hyper")] -pub mod hyper; - macro_rules! try_opt ( ($expr:expr) => ( match $expr { @@ -39,51 +33,49 @@ macro_rules! try_opt ( ) ); +mod boundary; + +#[cfg(feature = "hyper")] +pub mod hyper; + +#[cfg(feature = "iron")] +pub mod iron; + +const RANDOM_FILENAME_LEN: usize = 12; + /// The server-side implementation of `multipart/form-data` requests. /// -/// Implements `Borrow` to allow access to the request object. +/// Implements `Borrow` to allow access to the request body, if desired. pub struct Multipart { source: BoundaryReader, - line_buf: String, - /// The directory for saving files in this request. - /// By default, this is set to a subdirectory of `std::env::temp_dir()` with a - /// random alphanumeric name. - pub save_dir: PathBuf, + line_buf: String, } - impl Multipart<()> { /// If the given `R: HttpRequest` is a POST request of `Content-Type: multipart/form-data`, /// return the wrapped request as `Ok(Multipart)`, otherwise `Err(R)`. pub fn from_request(req: R) -> Result, R> { //FIXME: move `map` expr to `Some` arm when nonlexical borrow scopes land. - let boundary = match req.multipart_boundary().map(|b| format!("--{}", b)){ + let boundary = match req.multipart_boundary().map(String::from) { Some(boundary) => boundary, None => return Err(req), }; - debug!("Boundary: {}", boundary); - - Ok( - Multipart { - source: BoundaryReader::from_reader(req.body(), boundary), - line_buf: String::new(), - save_dir: ::temp_dir(), - } - ) + Ok(Multipart::with_body(req.body(), boundary)) } } impl Multipart { - pub fn with_body(body: B, boundary: &str) -> Self { - let boundary = format!("--{}", boundary); + /// Construct a new `Multipart` with the given body reader and boundary. + /// This will prepend the requisite `"--"` to the boundary. + pub fn with_body>(body: B, boundary: Bnd) -> Self { + let boundary = prepend_str("--", boundary.into()); debug!("Boundary: {}", boundary); Multipart { source: BoundaryReader::from_reader(body, boundary), line_buf: String::new(), - save_dir: ::temp_dir(), } } @@ -128,29 +120,54 @@ impl Multipart { Ok(ContentType::read_from(line)) } - /// Read the request fully, parsing all fields and saving all files in `self.save_dir`. + /// Read the request fully, parsing all fields and saving all files in a new temporary + /// directory. /// /// If there is an error in reading the request, returns the partial result along with the - /// error. - pub fn save_all(&mut self) -> (Entries, Option) { - let mut entries = Entries::with_path(self.save_dir.clone()); + /// error. See [`SaveResult`](../enum.saveresult.html) for more information. + pub fn save_all(&mut self) -> SaveResult { + let mut entries = match Entries::new_tempdir() { + Ok(entries) => entries, + Err(err) => return SaveResult::Error(err), + }; + + match self.read_to_entries(&mut entries) { + Ok(()) => SaveResult::Full(entries), + Err(err) => SaveResult::Partial(entries, err), + } + } - loop { - match self.read_entry() { - Ok(Some(field)) => match field.data { - MultipartData::File(mut file) => { - entries.files.insert(field.name, file.save()); - }, - MultipartData::Text(text) => { - entries.fields.insert(field.name, text.into()); - }, + /// Read the request fully, parsing all fields and saving all files in a new temporary + /// directory under `dir`. + /// + /// If there is an error in reading the request, returns the partial result along with the + /// error. See [`SaveResult`](../enum.saveresult.html) for more information. + pub fn save_all_under>(&mut self, dir: P) -> SaveResult { + let mut entries = match Entries::new_tempdir_in(dir) { + Ok(entries) => entries, + Err(err) => return SaveResult::Error(err), + }; + + match self.read_to_entries(&mut entries) { + Ok(()) => SaveResult::Full(entries), + Err(err) => SaveResult::Partial(entries, err), + } + } + + fn read_to_entries(&mut self, entries: &mut Entries) -> io::Result<()> { + while let Some(field) = try!(self.read_entry()) { + match field.data { + MultipartData::File(mut file) => { + let file = try!(file.save_in(&entries.dir)); + entries.files.insert(field.name, file); + }, + MultipartData::Text(text) => { + entries.fields.insert(field.name, text.into()); }, - Ok(None) => break, - Err(err) => return (entries, Some(err)), } } - (entries, None) + Ok(()) } fn read_line(&mut self) -> io::Result<&str> { @@ -195,6 +212,53 @@ impl Borrow for Multipart { } } +/// The result of [`Multipart::save_all()`](../struct.multipart.html#method.save_all). +#[derive(Debug)] +pub enum SaveResult { + /// The operation was a total success. Contained are all entries of the request. + Full(Entries), + /// The operation errored partway through. Contained are the entries gathered thus far, + /// as well as the error that ended the process. + Partial(Entries, io::Error), + /// The `TempDir` for `Entries` could not be constructed. Contained is the error detailing the + /// problem. + Error(io::Error), +} + +impl SaveResult { + /// Take the `Entries` from `self`, if applicable, and discarding + /// the error, if any. + pub fn to_entries(self) -> Option { + use self::SaveResult::*; + + match self { + Full(entries) | Partial(entries, _) => Some(entries), + Error(_) => None, + } + } + + /// Decompose `self` to `(Option, Option)` + pub fn to_opt(self) -> (Option, Option) { + use self::SaveResult::*; + + match self { + Full(entries) => (Some(entries), None), + Partial(entries, error) => (Some(entries), Some(error)), + Error(error) => (None, Some(error)), + } + } + + /// Map `self` to an `io::Result`, discarding the error in the `Partial` case. + pub fn to_result(self) -> io::Result { + use self::SaveResult::*; + + match self { + Full(entries) | Partial(entries, _) => Ok(entries), + Error(error) => Err(error), + } + } +} + struct ContentType { val: Mime, #[allow(dead_code)] @@ -282,6 +346,9 @@ fn get_remainder_after<'a>(needle: &str, haystack: &'a str) -> Option<(&'a str)> } /// A server-side HTTP request that may or may not be multipart. +/// +/// May be implemented by mutable references if providing the request or body by-value is +/// undesirable. pub trait HttpRequest { /// The body of this request. type Body: Read; @@ -289,10 +356,10 @@ pub trait HttpRequest { /// with the `Content-Type` header set to `multipart/form-data`. /// /// The boundary string should be supplied as an extra value of the `Content-Type` header, e.g. - /// `Content-Type: multipart/form-data; boundary=myboundary`. + /// `Content-Type: multipart/form-data; boundary={boundary}`. fn multipart_boundary(&self) -> Option<&str>; - /// Return a mutable reference to the request body for reading. + /// Return the request body for reading. fn body(self) -> Self::Body; } @@ -320,18 +387,14 @@ impl<'a, B: Read + 'a> MultipartField<'a, B> { MultipartFile::from_stream( cont_disp.filename, content_type.val, - &multipart.save_dir, &mut multipart.source, ) ) }, None => { // Empty line consumed by read_content_type() - let text = try!(multipart.read_to_string()); - // The last two characters are "\r\n". - // We can't do a simple trim because the content might be terminated - // with line separators we want to preserve. - MultipartData::Text(&text[..text.len()]) + let text = try!(multipart.read_to_string()); + MultipartData::Text(&text) }, }; @@ -386,64 +449,102 @@ impl<'a, B> MultipartData<'a, B> { pub struct MultipartFile<'a, B: 'a> { filename: Option, content_type: Mime, - save_dir: &'a Path, stream: &'a mut BoundaryReader, } impl<'a, B: Read> MultipartFile<'a, B> { fn from_stream(filename: Option, content_type: Mime, - save_dir: &'a Path, stream: &'a mut BoundaryReader) -> MultipartFile<'a, B> { MultipartFile { filename: filename, content_type: content_type, - save_dir: save_dir, stream: stream, } } + /// Save this file to the given output stream. + /// + /// If successful, returns the number of bytes written. + /// + /// Retries when `io::Error::kind() == io::ErrorKind::Interrupted`. + pub fn save_to(&mut self, mut out: W) -> io::Result { + retry_on_interrupt(|| io::copy(self.stream, &mut out)) + } + + /// Save this file to the given output stream, truncated to `limit` + /// (no more than `limit` bytes will be written out). + /// + /// If successful, returns the number of bytes written. + /// + /// Retries when `io::Error::kind() == io::ErrorKind::Interrupted`. + pub fn save_to_limited(&mut self, mut out: W, limit: u64) -> io::Result { + retry_on_interrupt(|| io::copy(&mut self.stream.take(limit), &mut out)) + } + /// Save this file to `path`. /// - /// Returns the number of bytes written on success, or any errors otherwise. + /// Returns the saved file info on success, or any errors otherwise. /// /// Retries when `io::Error::kind() == io::ErrorKind::Interrupted`. - pub fn save_as(&mut self, path: &Path) -> io::Result { - let mut file = try!(File::create(path)); - retry_on_interrupt(|| io::copy(self.stream, &mut file)) + pub fn save_as>(&mut self, path: P) -> io::Result { + let path = path.into(); + let file = try!(create_full_path(&path)); + let size = try!(self.save_to(file)); + + Ok(SavedFile { + path: path, + filename: self.filename.clone(), + size: size, + }) } /// Save this file in the directory pointed at by `dir`, - /// using `self.filename()` if present, or a random alphanumeric string otherwise. + /// using a random alphanumeric string as the filename. /// /// Any missing directories in the `dir` path will be created. /// - /// `self.filename()` is sanitized of all file separators before being appended to `dir`. - /// - /// Returns the created file's path on success, or any errors otherwise. + /// Returns the saved file's info on success, or any errors otherwise. /// /// Retries when `io::Error::kind() == io::ErrorKind::Interrupted`. - pub fn save_in(&mut self, dir: &Path) -> io::Result { - try!(fs::create_dir_all(dir)); - let path = self.gen_safe_file_path(dir); - self.save_as(&path).map(move |_| path) + pub fn save_in>(&mut self, dir: P) -> io::Result { + let path = dir.as_ref().join(::random_alphanumeric(RANDOM_FILENAME_LEN)); + self.save_as(path) } - /// Save this file in the directory pointed at by `self.save_dir`, - /// using `self.filename()` if present, or a random alphanumeric string otherwise. - /// - /// Any missing directories in the `self.save_dir` path will be created. + /// Save this file to `path`, truncated to `limit` (no more than `limit` bytes will be written out). /// - /// `self.filename()` is sanitized of all file separators before being appended to `self.save_dir`. + /// Any missing directories in the `dir` path will be created. /// - /// Returns the created file's path on success, or any errors otherwise. + /// Returns the saved file's info on success, or any errors otherwise. /// /// Retries when `io::Error::kind() == io::ErrorKind::Interrupted`. - pub fn save(&mut self) -> io::Result { - try!(fs::create_dir_all(self.save_dir)); - let path = self.gen_safe_file_path(self.save_dir); - self.save_as(&path).map(move |_| path) + pub fn save_as_limited>(&mut self, path: P, limit: u64) -> io::Result { + let path = path.into(); + let file = try!(create_full_path(&path)); + let size = try!(self.save_to_limited(file, limit)); + + Ok(SavedFile { + path: path, + filename: self.filename.clone(), + size: size, + }) } + + /// Save this file in the directory pointed at by `dir`, + /// using a random alphanumeric string as the filename. + /// + /// Truncates file to `limit` (no more than `limit` bytes will be written out). + /// + /// Any missing directories in the `dir` path will be created. + /// + /// Returns the saved file's info on success, or any errors otherwise. + /// + /// Retries when `io::Error::kind() == io::ErrorKind::Interrupted`. + pub fn save_in_limited>(&mut self, dir: P, limit: u64) -> io::Result { + let path = dir.as_ref().join(::random_alphanumeric(RANDOM_FILENAME_LEN)); + self.save_as_limited(path, limit) + } /// Get the filename of this entry, if supplied. /// @@ -457,23 +558,8 @@ impl<'a, B: Read> MultipartFile<'a, B> { /// Get the MIME type (`Content-Type` value) of this file, if supplied by the client, /// or `"applicaton/octet-stream"` otherwise. - pub fn content_type(&self) -> Mime { - self.content_type.clone() - } - - /// The save directory assigned to this file field by the `Multipart` instance it was read - /// from. - pub fn save_dir(&self) -> &Path { - self.save_dir - } - - fn gen_safe_file_path(&self, dir: &Path) -> PathBuf { - self.filename().map(Path::new) - .and_then(Path::file_name) //Make sure there's no path separators in the filename - .map_or_else( - || dir.join(::random_alphanumeric(8)), - |filename| dir.join(filename), - ) + pub fn content_type(&self) -> &Mime { + &self.content_type } } @@ -494,25 +580,130 @@ impl<'a, B: Read> BufRead for MultipartFile<'a, B> { } /// A result of `Multipart::save_all()`. +#[derive(Debug)] pub struct Entries { - /// The text fields of the multipart request. + /// The text fields of the multipart request, mapped by field name -> value. pub fields: HashMap, - /// A map of file field names to their save results. - pub files: HashMap>, - /// The directory the files were saved under. - pub dir: PathBuf, + /// A map of file field names to their contents saved on the filesystem. + pub files: HashMap, + /// The directory the files in this request were saved under; may be temporary or permanent. + pub dir: SaveDir, } impl Entries { - fn with_path>(path: P) -> Entries { + fn new_tempdir_in>(path: P) -> io::Result { + TempDir::new_in(path, "multipart").map(Self::with_tempdir) + } + + fn new_tempdir() -> io::Result { + TempDir::new("multipart").map(Self::with_tempdir) + } + + fn with_tempdir(tempdir: TempDir) -> Entries { Entries { fields: HashMap::new(), files: HashMap::new(), - dir: path.into(), + dir: SaveDir::Temp(tempdir), + } + } +} + +/// The save directory for `Entries`. May be temporary (delete-on-drop) or permanent. +pub enum SaveDir { + /// This directory is temporary and will be deleted, along with its contents, when dropped. + Temp(TempDir), + /// This directory is permanent and will be left on the filesystem. + Perm(PathBuf), +} + +impl SaveDir { + /// Get the path of this directory, either temporary or permanent. + pub fn as_path(&self) -> &Path { + use self::SaveDir::*; + match *self { + Temp(ref tempdir) => tempdir.path(), + Perm(ref pathbuf) => &*pathbuf, + } + } + + /// Returns `true` if this is a temporary directory which will be deleted on-drop. + pub fn is_temporary(&self) -> bool { + use self::SaveDir::*; + match *self { + Temp(_) => true, + Perm(_) => false, + } + } + + /// If this `SaveDir` is temporary, convert it to permanent. + /// This is a no-op if it already is permanent. + /// + /// ##Note + /// Even though this will prevent deletion on-drop, the temporary folder on most OSes + /// (where this directory is created by default) can be automatically cleared by the OS at any + /// time, usually on reboot or when free space is low. + pub fn keep(&mut self) { + use self::SaveDir::*; + *self = match mem::replace(self, Perm(PathBuf::new())) { + Temp(tempdir) => Perm(tempdir.into_path()), + old_self => old_self, + }; + } + + /// Delete this directory and its contents, regardless of its permanence. + /// + /// ###Warning + /// This is very likely irreversible, depending on the OS implementation. + /// + /// Files deleted programmatically are deleted directly from disk, as compared to most file + /// manager applications which use a staging area from which deleted files can be safely + /// recovered (i.e. Windows' Recycle Bin, OS X's Trash Can, etc.). + pub fn delete(self) -> io::Result<()> { + use self::SaveDir::*; + match self { + Temp(tempdir) => tempdir.close(), + Perm(pathbuf) => fs::remove_dir_all(&pathbuf), + } + } +} + +impl AsRef for SaveDir { + fn as_ref(&self) -> &Path { + self.as_path() + } +} + +// grrr, no Debug impl for TempDir, can't derive +// FIXME when this lands: https://github.com/rust-lang-nursery/tempdir/pull/9 +impl fmt::Debug for SaveDir { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + use self::SaveDir::*; + + match *self { + Temp(ref tempdir) => write!(f, "SaveDir::Temp({:?})", tempdir.path()), + Perm(ref path) => write!(f, "SaveDir::Perm({:?})", path), } } } +/// A file saved to the local filesystem from a multipart request. +#[derive(Debug)] +pub struct SavedFile { + /// The complete path this file was saved at. + pub path: PathBuf, + + /// The original filename of this file, if one was provided in the request. + /// + /// ##Warning + /// You should treat this value as untrustworthy because it is an arbitrary string provided by + /// the client. You should *not* blindly append it to a directory path and save the file there, + /// as such behavior could easily be exploited by a malicious client. + pub filename: Option, + + /// The number of bytes written to the disk; may be truncated. + pub size: u64, +} + fn retry_on_interrupt(mut do_fn: F) -> io::Result where F: FnMut() -> io::Result { loop { match do_fn() { @@ -524,3 +715,32 @@ fn retry_on_interrupt(mut do_fn: F) -> io::Result where F: FnMut() -> i } } +fn prepend_str(prefix: &str, mut string: String) -> String { + string.reserve(prefix.len()); + + unsafe { + let bytes = string.as_mut_vec(); + + // This addition is safe because it was already done in `String::reserve()` + // which would have panicked if it overflowed. + let old_len = bytes.len(); + let new_len = bytes.len() + prefix.len(); + bytes.set_len(new_len); + + ptr::copy(bytes.as_ptr(), bytes[prefix.len()..].as_mut_ptr(), old_len); + ptr::copy(prefix.as_ptr(), bytes.as_mut_ptr(), prefix.len()); + } + + string +} + +fn create_full_path(path: &Path) -> io::Result { + if let Some(parent) = path.parent() { + try!(fs::create_dir_all(parent)); + } else { + // RFC: return an error instead? + warn!("Attempting to save file in what looks like a root directory. File path: {:?}", path); + } + + File::create(&path) +} From 47004554ceb11b6009b76527e9f9161b3e6be94d Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Mon, 7 Mar 2016 00:23:37 -0800 Subject: [PATCH 107/453] Implement iron::Intercept --- multipart/src/server/iron.rs | 218 ++++++++++++++++++++++++++++++++--- 1 file changed, 204 insertions(+), 14 deletions(-) diff --git a/multipart/src/server/iron.rs b/multipart/src/server/iron.rs index 75a3af6b1..8d18f3244 100644 --- a/multipart/src/server/iron.rs +++ b/multipart/src/server/iron.rs @@ -1,23 +1,22 @@ //! Adaptor types and impls for `iron::Request`. Includes a `BeforeMiddleware` implementation. use iron::headers::ContentType; -use iron::BeforeMiddleware; use iron::mime::{Mime, TopLevel, SubLevel}; use iron::request::{Body, Request}; +use iron::typemap::Key; +use iron::{BeforeMiddleware, IronError, IronResult}; -use tempdir::TempDir; - -use std::collections::HashMap; use std::path::PathBuf; +use std::{error, fmt}; -use super::{HttpRequest, Multipart}; +use super::{Entries, HttpRequest, Multipart, MultipartData}; impl<'r, 'a, 'b> HttpRequest for &'r mut Request<'a, 'b> { type Body = &'r mut Body<'a, 'b>; fn multipart_boundary(&self) -> Option<&str> { let content_type = try_opt!(self.headers.get::()); - if let Mime(TopLevel::Multipart, SubLevel::FormData, _) = *content_type { + if let Mime(TopLevel::Multipart, SubLevel::FormData, _) = **content_type { content_type.get_param("boundary").map(|b| b.as_str()) } else { None @@ -33,30 +32,132 @@ impl<'r, 'a, 'b> HttpRequest for &'r mut Request<'a, 'b> { pub const DEFAULT_FILE_SIZE_LIMIT: u64 = 2 * 1024 * 1024; /// The default file count limit for `Intercept`. -pub const DEFAULT_FILE_COUNT_LIMIT: u64 = 16; +pub const DEFAULT_FILE_COUNT_LIMIT: u32 = 16; +/// A `BeforeMiddleware` for Iron which will intercept and read-out multipart requests. +/// +/// Any errors during which occur during reading will be passed on as `IronError`. +#[derive(Debug)] pub struct Intercept { + /// The parent directory for all temporary directories created by this middleware. + /// Will be created if it doesn't exist. + /// + /// If omitted, uses the OS temporary directory. + /// + /// Default value: `None`. pub temp_dir_path: Option, + /// The size limit of uploaded files, in bytes. + /// + /// Files which exceed this size will be rejected. + /// See the `limit_behavior` field for more info. + /// + /// Default value: [`DEFAULT_FILE_SIZE_LIMIT`](../constant.default_file_size_limit.html) pub file_size_limit: u64, - pub file_count_limit: u64, + /// The limit on the number of files which will be saved from + /// the request. Requests which exceed this count will be rejected. + /// + /// Default value: [`DEFAULT_FILE_COUNT_LIMT`](../constant.default_file_count_limit.html) + pub file_count_limit: u32, + /// What to do when a file count or size limit has been exceeded. + /// + /// See [`LimitBehavior`](../enum.limitbehavior.html) + pub limit_behavior: LimitBehavior, } impl Intercept { + /// Create a new `Intercept` with default values. pub fn new() -> Self { Default::default() } + /// Set the `temp_dir_path` for this middleware. pub fn temp_dir_path>(self, path: P) -> Self { - Intercept { temp_dir_path: path.into(), .. self } + Intercept { temp_dir_path: Some(path.into()), .. self } } + /// Set the `file_size_limit` for this middleware. pub fn file_size_limit(self, limit: u64) -> Self { Intercept { file_size_limit: limit, .. self } } - pub fn file_count_limit(self, limit: u64) -> Self { + /// Set the `file_count_limit` for this middleware. + pub fn file_count_limit(self, limit: u32) -> Self { Intercept { file_count_limit: limit, .. self } } + + /// Set the `limit_behavior` for this middleware. + pub fn limit_behavior(self, behavior: LimitBehavior) -> Self { + Intercept { limit_behavior: behavior, .. self } + } + + fn read_request(&self, req: &mut Request) -> IronResult> { + macro_rules! try_iron( + ($try:expr; $($fmt_args:tt)+) => ( + match $try { + Ok(ok) => ok, + Err(err) => return Err(IronError::new(err, format!($($fmt_args)*))), + } + ); + ($try:expr; $err_msg:expr) => ( + match $try { + Ok(ok) => ok, + Err(err) => return Err(IronError::new(err, $err_msg)), + } + ); + ($try:expr) => ( + try_iron!($try; ""); + ) + ); + + let mut multipart = match Multipart::from_request(req) { + Ok(multipart) => multipart, + Err(_) => return Ok(None), + }; + + let mut entries = try_iron!( + self.temp_dir_path.as_ref() + .map_or_else(Entries::new_tempdir, Entries::new_tempdir_in); + "Error opening temporary directory for request." + ); + + let mut file_count = 0; + + while let Some(field) = try_iron!(multipart.read_entry()) { + match field.data { + MultipartData::File(mut file) => { + if self.limit_behavior.throw_error() && file_count >= self.file_count_limit { + return Err(FileCountLimitError(self.file_count_limit).into()); + } + + let file = try_iron!( + file.save_in_limited(&entries.dir, self.file_size_limit); + "Error reading field: \"{}\" (filename: \"{}\")", + field.name, + file.filename.as_ref().map_or("(none)", String::as_ref) + ); + + if file.size == self.file_size_limit { + if self.limit_behavior.throw_error() { + return Err(FileSizeLimitError::new(field.name, file.filename).into()); + } else { + warn!( + "File size limit reached for field {:?} (filename: {:?})", + field.name, file.filename + ); + } + } + + entries.files.insert(field.name, file); + file_count += 1; + }, + MultipartData::Text(text) => { + entries.fields.insert(field.name, text.into()); + }, + } + } + + Ok(Some(entries)) + } } impl Default for Intercept { @@ -65,22 +166,111 @@ impl Default for Intercept { temp_dir_path: None, file_size_limit: DEFAULT_FILE_SIZE_LIMIT, file_count_limit: DEFAULT_FILE_COUNT_LIMIT, + limit_behavior: LimitBehavior::ThrowError, } } } impl BeforeMiddleware for Intercept { fn before(&self, req: &mut Request) -> IronResult<()> { - + try!(self.read_request(req)) + .map(|entries| req.extensions.insert::(entries)); + + Ok(()) } } +impl Key for Entries { + type Value = Self; +} + +/// The behavior of `Intercept` when a file size or count limit is exceeded. +#[derive(Clone, Copy, Debug)] +#[repr(u32)] pub enum LimitBehavior { + /// Return an error from the middleware describing the issue. ThrowError, + /// Ignore the limit. + /// + /// In the case of file size limits, the offending file will be truncated + /// in the result. + /// + /// In the case of file count limits, the request will be completed. Continue, } -struct LimitReader { - inner: R, - limit: u64, +impl LimitBehavior { + fn throw_error(self) -> bool { + use self::LimitBehavior::*; + + match self { + ThrowError => true, + Continue => false, + } + } +} + +/// An error returned from `Intercept` when the size limit +/// for an individual file is exceeded. +#[derive(Debug)] +pub struct FileSizeLimitError { + /// The field where the error occurred. + pub field: String, + /// The filename of the oversize file, if it was provided. + pub filename: Option, +} + +impl FileSizeLimitError { + fn new(field: String, filename: Option) -> Self { + FileSizeLimitError { + field: field, + filename: filename + } + } +} + +impl error::Error for FileSizeLimitError { + fn description(&self) -> &str { + "file size limit reached" + } +} + +impl fmt::Display for FileSizeLimitError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self.filename { + Some(ref filename) => write!(f, "File size limit reached for field \"{}\" (filename: \"{}\")", self.field, filename), + None => write!(f, "File size limit reached for field \"{}\" (no filename)", self.field), + } + } +} + +impl Into for FileSizeLimitError { + fn into(self) -> IronError { + let desc_str = self.to_string(); + IronError::new(self, desc_str) + } +} + +/// An error returned from `Intercept` when the file count limit +/// for a single request was exceeded. +#[derive(Debug)] +pub struct FileCountLimitError(u32); + +impl error::Error for FileCountLimitError { + fn description(&self) -> &str { + "file count limit reached" + } +} + +impl fmt::Display for FileCountLimitError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "File count limit reached for request. Limit: {}", self.0) + } +} + +impl Into for FileCountLimitError { + fn into(self) -> IronError { + let desc_string = self.to_string(); + IronError::new(self, desc_string) + } } From 4f245b5463c034dea51f8c004137701b1751bdd9 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Mon, 7 Mar 2016 00:30:46 -0800 Subject: [PATCH 108/453] Bootstrap nickel --- multipart/Cargo.toml | 39 ++++++++++++++++++++-------------- multipart/src/lib.rs | 3 +++ multipart/src/server/mod.rs | 8 +++++++ multipart/src/server/nickel.rs | 19 +++++++++++++++++ 4 files changed, 53 insertions(+), 16 deletions(-) create mode 100644 multipart/src/server/nickel.rs diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index c891b69f2..b05c60bdc 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -1,44 +1,51 @@ [package] - name = "multipart" + version = "0.5.0" + authors = ["Austin Bonander "] description = "A backend-agnostic extension for HTTP libraries that provides support for POST multipart/form-data requests on for both client and server." -repository = "http://github.com/cybergeek94/multipart" - documentation = "http://rust-ci.org/cybergeek94/multipart/doc/multipart/" -license = "MIT OR Apache-2.0" - keywords = ["form-data", "hyper", "iron", "http", "post", "upload"] -[features] -server = ["buf_redux", "memchr"] -client = [] -default = ["hyper", "server", "client"] +repository = "http://github.com/cybergeek94/multipart" + +license = "MIT OR Apache-2.0" [dependencies] -log = "0.3" env_logger = "0.3" +log = "0.3" mime = "0.1" mime_guess = "1.4" rand = "0.3" tempdir = "0.3" -[dependencies.hyper] -version = "0.7" +[dependencies.buf_redux] optional = true +version = "0.1" -[dependencies.iron] -version = "0.2" +[dependencies.hyper] optional = true +version = "0.7" -[dependencies.buf_redux] -version = "0.1" +[dependencies.iron] optional = true +version = "0.2" [dependencies.memchr] +optional = true version = "0.1" + +[dependencies.nickel] optional = true +version = "0.7.3" + +[features] +client = [] +default = ["hyper", "server", "client"] +server = ["buf_redux", "memchr"] +#Disabled until Nickel upgrades to Hyper 0.7 +#nickel2 = ["hyper", "nickel"] diff --git a/multipart/src/lib.rs b/multipart/src/lib.rs index deb85ff21..afadb8154 100644 --- a/multipart/src/lib.rs +++ b/multipart/src/lib.rs @@ -32,6 +32,9 @@ extern crate hyper; #[cfg(feature = "iron")] extern crate iron; +#[cfg(feature = "nickel")] +extern crate nickel; + #[cfg(feature = "tiny_http")] extern crate tiny_http; diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index d0274137c..08e987910 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -41,6 +41,14 @@ pub mod hyper; #[cfg(feature = "iron")] pub mod iron; +#[cfg(feature = "nickel2")] +pub mod nickel; + +/* FIXME: when tiny_http gains proper header parsing. +#[cfg(feature = "tiny_http")] +pub mod tiny_http; +*/ + const RANDOM_FILENAME_LEN: usize = 12; /// The server-side implementation of `multipart/form-data` requests. diff --git a/multipart/src/server/nickel.rs b/multipart/src/server/nickel.rs new file mode 100644 index 000000000..1bcf3a38c --- /dev/null +++ b/multipart/src/server/nickel.rs @@ -0,0 +1,19 @@ +//! Trait impl for Nickel, piggybacking on Hyper's integration. + +use nickel::Request; + +use hyper::server::Request as HyperRequest; + +use super::HttpRequest; + +impl<'r, 'mw, 'server, D: 'mw> HttpRequest for &'r Request<'mw, 'server, D> { + type Body = &'r mut HyperRequest<'mw, 'server>; + + fn multipart_boundary(&self) -> Option<&str> { + self.origin.multipart_boundary() + } + + fn body(self) -> Self::Body { + &mut self.origin + } +} From f77c6323fe99f0765507f9a4540e68245df7dea2 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sat, 5 Mar 2016 13:42:03 -0800 Subject: [PATCH 109/453] Update local_test --- multipart/src/local_test.rs | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/multipart/src/local_test.rs b/multipart/src/local_test.rs index 16c96e8fd..01b807f0f 100644 --- a/multipart/src/local_test.rs +++ b/multipart/src/local_test.rs @@ -16,6 +16,7 @@ use std::collections::HashMap; use std::io; use std::io::prelude::*; +#[derive(Debug)] struct TestFields { texts: HashMap, files: HashMap>, @@ -108,10 +109,21 @@ fn test_server(buf: HttpBuffer, mut fields: TestFields) { let mut multipart = Multipart::from_request(buf.for_server()) .unwrap_or_else(|_| panic!("Buffer should be multipart!")); + trace!("Fields for server test: {:?}", fields); + while let Ok(Some(mut field)) = multipart.read_entry() { match field.data { MultipartData::Text(text) => { - let test_text = fields.texts.remove(&field.name).unwrap(); + let test_text = fields.texts.remove(&field.name); + + assert!( + test_text.is_some(), + "Got text field that wasn't in original dataset: {:?} : {:?} ", + field.name, text + ); + + let test_text = test_text.unwrap(); + assert!( text == test_text, "Unexpected data for field {:?}: Expected {:?}, got {:?}", @@ -215,6 +227,12 @@ impl<'a> Read for ServerBuffer<'a> { } impl<'a> ServerRequest for ServerBuffer<'a> { + type Body = Self; + fn multipart_boundary(&self) -> Option<&str> { Some(&self.boundary) } + + fn body(self) -> Self::Body { + self + } } From 9b1abfb3841190e02328838e657bafd6f295bc1e Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sat, 5 Mar 2016 16:20:29 -0800 Subject: [PATCH 110/453] Finalize tiny_http support --- multipart/Cargo.toml | 6 +++++- multipart/src/server/tiny_http.rs | 22 +++++++++++++++++++--- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index b05c60bdc..1a936a998 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -41,7 +41,11 @@ version = "0.1" [dependencies.nickel] optional = true -version = "0.7.3" +version = "0.7" + +[dependencies.tiny_http] +optional = true +version = "0.5" [features] client = [] diff --git a/multipart/src/server/tiny_http.rs b/multipart/src/server/tiny_http.rs index 13c1aa048..03c2c6f6c 100644 --- a/multipart/src/server/tiny_http.rs +++ b/multipart/src/server/tiny_http.rs @@ -1,13 +1,29 @@ -use tiny_http::Request; +//! Integration with [tiny_http](https://github.com/frewsxcv/tiny-http) with the `tiny_http` +//! feature (optional). +//! +//! Contains `impl `[`HttpRequest`](../trait.HttpRequest.html)` for tiny_http::Request` (not shown +//! here; see [`HttpRequest`'s implementors](../trait.HttpRequest.html#implementors)). + +pub use tiny_http::Request as TinyHttpRequest; use super::HttpRequest; use std::io::Read; -impl<'r> HttpRequest for &'r mut Request { +impl<'r> HttpRequest for &'r mut TinyHttpRequest { type Body = &'r mut Read; fn multipart_boundary(&self) -> Option<&str> { - self.headers.iter().find(|header| + const BOUNDARY: &'static str = "boundary="; + + let content_type = try_opt!(self.headers().iter().find(|header| header.field.equiv("Content-Type"))).value.as_str(); + let start = try_opt!(content_type.find(BOUNDARY)) + BOUNDARY.len(); + let end = content_type[start..].find(';').map_or(content_type.len(), |end| start + end); + + Some(&content_type[start .. end]) + } + + fn body(self) -> Self::Body { + self.as_reader() } } From 92fdb9ff8ed64179905aa3adb04d4e5a47ed2320 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sat, 5 Mar 2016 16:20:40 -0800 Subject: [PATCH 111/453] Documentation tweaks --- multipart/src/server/hyper.rs | 11 ++++-- multipart/src/server/iron.rs | 64 +++++++++++++++++++++++------------ multipart/src/server/mod.rs | 52 +++++++++++++++++----------- 3 files changed, 84 insertions(+), 43 deletions(-) diff --git a/multipart/src/server/hyper.rs b/multipart/src/server/hyper.rs index 3186d6801..b9c8a7d7e 100644 --- a/multipart/src/server/hyper.rs +++ b/multipart/src/server/hyper.rs @@ -6,11 +6,16 @@ // copied, modified, or distributed except according to those terms. //! Server-side integration with [Hyper](https://github.com/hyperium/hyper). //! Enabled with the `hyper` feature (on by default). +//! +//! Also contains an implementation of [`HttpRequest`](../trait.HttpRequest.html)` +//! for `hyper::server::Request` and `&mut hyper::server::Request`. use hyper::net::Fresh; use hyper::header::ContentType; use hyper::method::Method; use hyper::server::{Handler, Request, Response}; +pub use hyper::server::Request as HyperRequest; + use mime::{Mime, TopLevel, SubLevel, Attr, Value}; use super::{Multipart, HttpRequest}; @@ -67,7 +72,7 @@ where F: Fn(Multipart, Response), F: Send + Sync { } } -impl<'a, 'b> HttpRequest for Request<'a, 'b> { +impl<'a, 'b> HttpRequest for HyperRequest<'a, 'b> { type Body = Self; fn multipart_boundary(&self) -> Option<&str> { @@ -101,8 +106,8 @@ impl<'a, 'b> HttpRequest for Request<'a, 'b> { } } -impl<'r, 'a, 'b> HttpRequest for &'r mut Request<'a, 'b> { - type Body = &'r mut Request<'a, 'b>; +impl<'r, 'a, 'b> HttpRequest for &'r mut HyperRequest<'a, 'b> { + type Body = Self; fn multipart_boundary(&self) -> Option<&str> { if self.method != Method::Post { diff --git a/multipart/src/server/iron.rs b/multipart/src/server/iron.rs index 8d18f3244..a5c2039f5 100644 --- a/multipart/src/server/iron.rs +++ b/multipart/src/server/iron.rs @@ -1,8 +1,11 @@ -//! Adaptor types and impls for `iron::Request`. Includes a `BeforeMiddleware` implementation. +//! Integration with the [Iron](https://ironframework.io) framework, enabled with the `iron` feature (optional). Includes a `BeforeMiddleware` implementation. +//! +//! Not shown here: `impl `[`HttpRequest`](../trait.HttpRequest.html#implementors)` for +//! iron::Request`. use iron::headers::ContentType; use iron::mime::{Mime, TopLevel, SubLevel}; -use iron::request::{Body, Request}; +use iron::request::{Body as IronBody, Request as IronRequest}; use iron::typemap::Key; use iron::{BeforeMiddleware, IronError, IronResult}; @@ -11,8 +14,8 @@ use std::{error, fmt}; use super::{Entries, HttpRequest, Multipart, MultipartData}; -impl<'r, 'a, 'b> HttpRequest for &'r mut Request<'a, 'b> { - type Body = &'r mut Body<'a, 'b>; +impl<'r, 'a, 'b> HttpRequest for &'r mut IronRequest<'a, 'b> { + type Body = &'r mut IronBody<'a, 'b>; fn multipart_boundary(&self) -> Option<&str> { let content_type = try_opt!(self.headers.get::()); @@ -23,25 +26,49 @@ impl<'r, 'a, 'b> HttpRequest for &'r mut Request<'a, 'b> { } } - fn body(self) -> &'r mut Body<'a, 'b> { + fn body(self) -> &'r mut IronBody<'a, 'b> { &mut self.body } } -/// The default file size limit for `Intercept`, in bytes. +/// The default file size limit for [`Intercept`](struct.Intercept.html), in bytes. pub const DEFAULT_FILE_SIZE_LIMIT: u64 = 2 * 1024 * 1024; -/// The default file count limit for `Intercept`. +/// The default file count limit for [`Intercept`](struct.Intercept.html). pub const DEFAULT_FILE_COUNT_LIMIT: u32 = 16; -/// A `BeforeMiddleware` for Iron which will intercept and read-out multipart requests. +/// A `BeforeMiddleware` for Iron which will intercept and read-out multipart requests and store +/// the result in the request. +/// +/// Successful reads will be placed in the `extensions: TypeMap` field of `iron::Request` as an +/// [`Entries`](../struct.Entries.html) instance (as both key-type and value): +/// +/// ```no_run +/// extern crate iron; +/// extern crate multipart; +/// +/// use iron::prelude::*; +/// +/// use multipart::server::Entries; +/// use multipart::server::iron::Intercept; +/// +/// fn main() { +/// Iron::new(Chain::new(|req: &mut Request| if let Some(entries) = +/// req.extensions.get::() { +/// +/// Ok(Response::with(format!("{:?}", entries))) +/// } else { +/// Ok(Response::with("Not a multipart request")) +/// })).http("localhost:80").unwrap(); +/// } +/// ``` /// /// Any errors during which occur during reading will be passed on as `IronError`. #[derive(Debug)] pub struct Intercept { /// The parent directory for all temporary directories created by this middleware. - /// Will be created if it doesn't exist. - /// + /// Will be created if it doesn't exist (lazy). + /// /// If omitted, uses the OS temporary directory. /// /// Default value: `None`. @@ -51,25 +78,20 @@ pub struct Intercept { /// Files which exceed this size will be rejected. /// See the `limit_behavior` field for more info. /// - /// Default value: [`DEFAULT_FILE_SIZE_LIMIT`](../constant.default_file_size_limit.html) + /// Default value: [`DEFAULT_FILE_SIZE_LIMIT`](constant.default_file_size_limit.html) pub file_size_limit: u64, /// The limit on the number of files which will be saved from /// the request. Requests which exceed this count will be rejected. /// - /// Default value: [`DEFAULT_FILE_COUNT_LIMT`](../constant.default_file_count_limit.html) + /// Default value: [`DEFAULT_FILE_COUNT_LIMT`](constant.default_file_count_limit.html) pub file_count_limit: u32, /// What to do when a file count or size limit has been exceeded. /// - /// See [`LimitBehavior`](../enum.limitbehavior.html) + /// See [`LimitBehavior`](enum.limitbehavior.html) for more info. pub limit_behavior: LimitBehavior, } -impl Intercept { - /// Create a new `Intercept` with default values. - pub fn new() -> Self { - Default::default() - } - +impl Intercept { /// Set the `temp_dir_path` for this middleware. pub fn temp_dir_path>(self, path: P) -> Self { Intercept { temp_dir_path: Some(path.into()), .. self } @@ -90,7 +112,7 @@ impl Intercept { Intercept { limit_behavior: behavior, .. self } } - fn read_request(&self, req: &mut Request) -> IronResult> { + fn read_request(&self, req: &mut IronRequest) -> IronResult> { macro_rules! try_iron( ($try:expr; $($fmt_args:tt)+) => ( match $try { @@ -172,7 +194,7 @@ impl Default for Intercept { } impl BeforeMiddleware for Intercept { - fn before(&self, req: &mut Request) -> IronResult<()> { + fn before(&self, req: &mut IronRequest) -> IronResult<()> { try!(self.read_request(req)) .map(|entries| req.extensions.insert::(entries)); diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index 08e987910..311874ddd 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -22,7 +22,7 @@ use std::io::prelude::*; use std::path::{Path, PathBuf}; use std::{fmt, io, mem, ptr}; -pub use self::boundary::BoundaryReader; +use self::boundary::BoundaryReader; macro_rules! try_opt ( ($expr:expr) => ( @@ -42,12 +42,10 @@ pub mod hyper; pub mod iron; #[cfg(feature = "nickel2")] -pub mod nickel; +mod nickel; -/* FIXME: when tiny_http gains proper header parsing. #[cfg(feature = "tiny_http")] pub mod tiny_http; -*/ const RANDOM_FILENAME_LEN: usize = 12; @@ -60,8 +58,9 @@ pub struct Multipart { } impl Multipart<()> { - /// If the given `R: HttpRequest` is a POST request of `Content-Type: multipart/form-data`, - /// return the wrapped request as `Ok(Multipart)`, otherwise `Err(R)`. + /// If the given `HttpRequest` is a multipart/form-data POST request, + /// return the request body wrapped in the multipart reader. Otherwise, + /// returns the original request. pub fn from_request(req: R) -> Result, R> { //FIXME: move `map` expr to `Some` arm when nonlexical borrow scopes land. let boundary = match req.multipart_boundary().map(String::from) { @@ -129,10 +128,10 @@ impl Multipart { } /// Read the request fully, parsing all fields and saving all files in a new temporary - /// directory. + /// directory under the OS temporary directory. /// /// If there is an error in reading the request, returns the partial result along with the - /// error. See [`SaveResult`](../enum.saveresult.html) for more information. + /// error. See [`SaveResult`](enum.saveresult.html) for more information. pub fn save_all(&mut self) -> SaveResult { let mut entries = match Entries::new_tempdir() { Ok(entries) => entries, @@ -149,7 +148,7 @@ impl Multipart { /// directory under `dir`. /// /// If there is an error in reading the request, returns the partial result along with the - /// error. See [`SaveResult`](../enum.saveresult.html) for more information. + /// error. See [`SaveResult`](enum.saveresult.html) for more information. pub fn save_all_under>(&mut self, dir: P) -> SaveResult { let mut entries = match Entries::new_tempdir_in(dir) { Ok(entries) => entries, @@ -220,7 +219,7 @@ impl Borrow for Multipart { } } -/// The result of [`Multipart::save_all()`](../struct.multipart.html#method.save_all). +/// The result of [`Multipart::save_all()`](struct.multipart.html#method.save_all). #[derive(Debug)] pub enum SaveResult { /// The operation was a total success. Contained are all entries of the request. @@ -447,7 +446,7 @@ impl<'a, B> MultipartData<'a, B> { /// A representation of a file in HTTP `multipart/form-data`. /// -/// Note that the file is not yet saved to the system; +/// Note that the file is not yet saved to the local filesystem; /// instead, this struct exposes `Read` and `BufRead` impls which point /// to the beginning of the file's contents in the HTTP stream. /// @@ -480,7 +479,7 @@ impl<'a, B: Read> MultipartFile<'a, B> { retry_on_interrupt(|| io::copy(self.stream, &mut out)) } - /// Save this file to the given output stream, truncated to `limit` + /// Save this file to the given output stream, **truncated** to `limit` /// (no more than `limit` bytes will be written out). /// /// If successful, returns the number of bytes written. @@ -520,7 +519,7 @@ impl<'a, B: Read> MultipartFile<'a, B> { self.save_as(path) } - /// Save this file to `path`, truncated to `limit` (no more than `limit` bytes will be written out). + /// Save this file to `path`, **truncated** to `limit` (no more than `limit` bytes will be written out). /// /// Any missing directories in the `dir` path will be created. /// @@ -542,7 +541,7 @@ impl<'a, B: Read> MultipartFile<'a, B> { /// Save this file in the directory pointed at by `dir`, /// using a random alphanumeric string as the filename. /// - /// Truncates file to `limit` (no more than `limit` bytes will be written out). + /// **Truncates** file to `limit` (no more than `limit` bytes will be written out). /// /// Any missing directories in the `dir` path will be created. /// @@ -618,9 +617,10 @@ impl Entries { /// The save directory for `Entries`. May be temporary (delete-on-drop) or permanent. pub enum SaveDir { - /// This directory is temporary and will be deleted, along with its contents, when dropped. + /// This directory is temporary and will be deleted, along with its contents, when this wrapper + /// is dropped. Temp(TempDir), - /// This directory is permanent and will be left on the filesystem. + /// This directory is permanent and will be left on the filesystem when this wrapper is dropped. Perm(PathBuf), } @@ -643,13 +643,27 @@ impl SaveDir { } } + /// Unwrap the `PathBuf` from `self`; if this is a temporary directory, + /// it will be converted to a permanent one. + pub fn into_path(self) -> PathBuf { + use self::SaveDir::*; + + match self { + Temp(tempdir) => tempdir.into_path(), + Perm(pathbuf) => pathbuf, + } + } + /// If this `SaveDir` is temporary, convert it to permanent. /// This is a no-op if it already is permanent. /// - /// ##Note + /// ###Warning: Potential Data Loss /// Even though this will prevent deletion on-drop, the temporary folder on most OSes /// (where this directory is created by default) can be automatically cleared by the OS at any /// time, usually on reboot or when free space is low. + /// + /// It is recommended that you relocate the files from a request which you want to keep to a + /// permanent folder on the filesystem. pub fn keep(&mut self) { use self::SaveDir::*; *self = match mem::replace(self, Perm(PathBuf::new())) { @@ -660,7 +674,7 @@ impl SaveDir { /// Delete this directory and its contents, regardless of its permanence. /// - /// ###Warning + /// ###Warning: Potential Data Loss /// This is very likely irreversible, depending on the OS implementation. /// /// Files deleted programmatically are deleted directly from disk, as compared to most file @@ -682,7 +696,7 @@ impl AsRef for SaveDir { } // grrr, no Debug impl for TempDir, can't derive -// FIXME when this lands: https://github.com/rust-lang-nursery/tempdir/pull/9 +// FIXME when tempdir > 0.3.4 is released (Debug PR landed 3/3/2016) impl fmt::Debug for SaveDir { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { use self::SaveDir::*; From 0d27dc6b7e5f8bdfb09a30d6dc241d732cbd433f Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sat, 5 Mar 2016 16:28:34 -0800 Subject: [PATCH 112/453] Fix iron example; remove superfluous macro variant --- multipart/src/server/iron.rs | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/multipart/src/server/iron.rs b/multipart/src/server/iron.rs index a5c2039f5..02310940d 100644 --- a/multipart/src/server/iron.rs +++ b/multipart/src/server/iron.rs @@ -53,13 +53,17 @@ pub const DEFAULT_FILE_COUNT_LIMIT: u32 = 16; /// use multipart::server::iron::Intercept; /// /// fn main() { -/// Iron::new(Chain::new(|req: &mut Request| if let Some(entries) = +/// let mut chain = Chain::new(|req: &mut Request| if let Some(entries) = /// req.extensions.get::() { /// /// Ok(Response::with(format!("{:?}", entries))) /// } else { /// Ok(Response::with("Not a multipart request")) -/// })).http("localhost:80").unwrap(); +/// }); +/// +/// chain.link_before(Intercept::default()); +/// +/// Iron::new(chain).http("localhost:80").unwrap(); /// } /// ``` /// @@ -120,12 +124,6 @@ impl Intercept { Err(err) => return Err(IronError::new(err, format!($($fmt_args)*))), } ); - ($try:expr; $err_msg:expr) => ( - match $try { - Ok(ok) => ok, - Err(err) => return Err(IronError::new(err, $err_msg)), - } - ); ($try:expr) => ( try_iron!($try; ""); ) From 4d643c1105449224a1a822e9179672ce16f372b7 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sat, 5 Mar 2016 16:57:35 -0800 Subject: [PATCH 113/453] Change eager client to returning errors immediately, export chain_result! --- multipart/src/client/lazy.rs | 2 + multipart/src/client/mod.rs | 71 +++++++++++------------------------- multipart/src/lib.rs | 31 +++++++++++++++- multipart/src/local_test.rs | 6 +-- 4 files changed, 57 insertions(+), 53 deletions(-) diff --git a/multipart/src/client/lazy.rs b/multipart/src/client/lazy.rs index 9521b5088..90df0b849 100644 --- a/multipart/src/client/lazy.rs +++ b/multipart/src/client/lazy.rs @@ -244,6 +244,7 @@ pub struct PreparedFields<'d> { // NOTE: the order of these fields have been reversed so fields can be popped one-by-one from // the end. fields: Vec>, + #[cfg_attr(not(feature = "hyper"), allow(dead_code))] boundary: String, content_len: Option, } @@ -340,6 +341,7 @@ impl<'d> PreparedFields<'d> { self.content_len } + #[cfg_attr(not(feature = "hyper"), allow(dead_code))] fn boundary(&self) -> &str { &self.boundary } diff --git a/multipart/src/client/mod.rs b/multipart/src/client/mod.rs index 73ffd651f..22dc9e9b3 100644 --- a/multipart/src/client/mod.rs +++ b/multipart/src/client/mod.rs @@ -28,17 +28,25 @@ pub use self::sized::SizedRequest; const BOUNDARY_LEN: usize = 16; +macro_rules! map_self { + ($selff:expr, $try:expr) => ( + match $try { + Ok(_) => Ok($selff), + Err(err) => Err(err.into()), + } + ) +} + /// The entry point of the client-side multipart API. /// /// Though they perform I/O, the `.write_*()` methods do not return `io::Result<_>` in order to /// facilitate method chaining. Upon the first error, all subsequent API calls will be no-ops until /// `.send()` is called, at which point the error will be reported. -pub struct Multipart { +pub struct Multipart { writer: MultipartWriter<'static, S>, - last_err: Option, } -impl Multipart { +impl Multipart<()> { /// Create a new `Multipart` to wrap a request. /// /// ## Returns Error @@ -48,38 +56,18 @@ impl Multipart { Ok(Multipart { writer: MultipartWriter::new(stream, boundary), - last_err: None, }) } } -impl Multipart { - /// Get a reference to the last error returned from writing to the HTTP stream, if any. - pub fn last_err(&self) -> Option<&S::Error> { - self.last_err.as_ref() - } - - /// Remove and return the last error to occur, allowing subsequent API calls to proceed - /// normally. - /// - /// ##Warning - /// If an error occurred during a write, the request body may be corrupt. - pub fn take_err(&mut self) -> Option { - self.last_err.take() - } - +impl Multipart { /// Write a text field to this multipart request. /// `name` and `val` can be either owned `String` or `&str`. /// /// ##Errors /// If something went wrong with the HTTP stream. - pub fn write_text, V: AsRef>(&mut self, name: N, val: V) -> &mut Self { - if self.last_err.is_none() { - self.last_err = self.writer.write_text(name.as_ref(), val.as_ref()) - .err().map(|err| err.into()) - } - - self + pub fn write_text, V: AsRef>(&mut self, name: N, val: V) -> Result<&mut Self, S::Error> { + map_self!(self, self.writer.write_text(name.as_ref(), val.as_ref())) } /// Open a file pointed to by `path` and write its contents to the multipart request, @@ -93,16 +81,11 @@ impl Multipart { /// ##Errors /// If there was a problem opening the file (was a directory or didn't exist), /// or if something went wrong with the HTTP stream. - pub fn write_file, P: AsRef>(&mut self, name: N, path: P) -> &mut Self { - if self.last_err.is_none() { - let name = name.as_ref(); - let path = path.as_ref(); - - self.last_err = self.writer.write_file(name, path).err() - .map(|err| err.into()); - } + pub fn write_file, P: AsRef>(&mut self, name: N, path: P) -> Result<&mut Self, S::Error> { + let name = name.as_ref(); + let path = path.as_ref(); - self + map_self!(self, self.writer.write_file(name, path)) } /// Write a byte stream to the multipart request as a file field, supplying `filename` if given, @@ -127,25 +110,15 @@ impl Multipart { // RFC: How to format this declaration? pub fn write_stream, St: Read>( &mut self, name: N, stream: &mut St, filename: Option<&str>, content_type: Option - ) -> &mut Self { - if self.last_err.is_none() { - let name = name.as_ref(); - - self.last_err = self.writer.write_stream(stream, name, filename, content_type) - .err().map(|err| err.into()); - } + ) -> Result<&mut Self, S::Error> { + let name = name.as_ref(); - self + map_self!(self, self.writer.write_stream(stream, name, filename, content_type)) } /// Finalize the request and return the response from the server, or the last error if set. pub fn send(self) -> Result { - match self.last_err { - None => { - try!(self.writer.finish()).finish() - }, - Some(err) => Err(err), - } + self.writer.finish().map_err(io::Error::into).and_then(|body| body.finish()) } } diff --git a/multipart/src/lib.rs b/multipart/src/lib.rs index afadb8154..44b7a233f 100644 --- a/multipart/src/lib.rs +++ b/multipart/src/lib.rs @@ -40,10 +40,39 @@ extern crate tiny_http; use rand::Rng; +/// Chain a series of results together, with or without previous results. +/// +/// ``` +/// #[macro_use] extern crate multipart; +/// +/// fn try_add_one(val: u32) -> Result { +/// if val < 5 { +/// Ok(val + 1) +/// } else { +/// Err(val) +/// } +/// } +/// +/// fn main() { +/// let res = chain_result! { +/// try_add_one(1), +/// prev -> try_add_one(prev), +/// prev -> try_add_one(prev), +/// prev -> try_add_one(prev) +/// }; +/// +/// println!("{:?}", res); +/// } +/// +/// ``` +#[macro_export] macro_rules! chain_result { ($first_expr:expr, $($try_expr:expr),*) => ( $first_expr $(.and_then(|_| $try_expr))* - ); + ); + ($first_expr:expr, $($($arg:ident),+ -> $try_expr:expr),*) => ( + $first_expr $(.and_then(|$($arg),+| $try_expr))* + ); } #[cfg(feature = "client")] diff --git a/multipart/src/local_test.rs b/multipart/src/local_test.rs index 01b807f0f..56636bf28 100644 --- a/multipart/src/local_test.rs +++ b/multipart/src/local_test.rs @@ -88,15 +88,15 @@ fn test_client(test_fields: &TestFields) -> HttpBuffer { // Intersperse file fields amongst text fields for (name, text) in &test_fields.texts { if let Some((file_name, file)) = test_files.next() { - multipart.write_stream(file_name, &mut &**file, None, None); + multipart.write_stream(file_name, &mut &**file, None, None).unwrap(); } - multipart.write_text(name, text); + multipart.write_text(name, text).unwrap(); } // Write remaining files for (file_name, file) in test_files { - multipart.write_stream(file_name, &mut &**file, None, None); + multipart.write_stream(file_name, &mut &**file, None, None).unwrap(); } From 9dc5add901c4947e3c40b87364663ac6845180b5 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Mon, 7 Mar 2016 00:54:24 -0800 Subject: [PATCH 114/453] Use alpha version --- multipart/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 1a936a998..8498014ad 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "multipart" -version = "0.5.0" +version = "0.5.0-alpha.1" authors = ["Austin Bonander "] From 7a66f00abf0d1a242a263b3b1a629775ce76c1a3 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Mon, 7 Mar 2016 01:28:55 -0800 Subject: [PATCH 115/453] Remove extra keyword (limit 5) --- multipart/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 8498014ad..0ee75a67e 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -9,7 +9,7 @@ description = "A backend-agnostic extension for HTTP libraries that provides sup documentation = "http://rust-ci.org/cybergeek94/multipart/doc/multipart/" -keywords = ["form-data", "hyper", "iron", "http", "post", "upload"] +keywords = ["form-data", "hyper", "iron", "http", "upload"] repository = "http://github.com/cybergeek94/multipart" From 36150b00bf89a712d6c058264adbda14eaecc9ef Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sat, 12 Mar 2016 21:36:13 -0800 Subject: [PATCH 116/453] Update README --- multipart/README.md | 95 ++++++++++++--------------------------------- 1 file changed, 25 insertions(+), 70 deletions(-) diff --git a/multipart/README.md b/multipart/README.md index da1be8459..33cd8ff24 100644 --- a/multipart/README.md +++ b/multipart/README.md @@ -1,77 +1,32 @@ -Multipart + Hyper [![Build Status](https://travis-ci.org/cybergeek94/multipart.svg?branch=master)](https://travis-ci.org/cybergeek94/multipart) [![On Crates.io](https://img.shields.io/crates/v/multipart.svg)](https://crates.io/crates/multipart) +Multipart [![Build Status](https://travis-ci.org/cybergeek94/multipart.svg?branch=master)](https://travis-ci.org/cybergeek94/multipart) [![On Crates.io](https://img.shields.io/crates/v/multipart.svg)](https://crates.io/crates/multipart) ========= Client- and server-side abstractions for HTTP file uploads (POST requests with `Content-Type: multipart/form-data`). -Provides integration with [Hyper](https://github.com/hyperium/hyper) via the `hyper` feature. More to come! - -####[Documentation](http://rust-ci.org/cybergeek94/multipart/doc/multipart/) - -Usage ------ - -In your `Cargo.toml`: -```toml -# Currently only useful with `hyper` and `url` crates: -[dependencies] -hyper = "*" -url = "*" - -[dependencies.multipart] -version = "*" # Or use the version in the Crates.io badge above. -# You can also select which features to compile: -# default-features = false -# features = ["hyper", "server", "client"] -``` - -Client-side example using Hyper (`features = ["hyper", "client"]` or default): -```rust -extern crate hyper; -extern crate multipart; -extern crate url; - -use hyper::client::request::Request; -use hyper::method::Method; - -use multipart::client::Multipart; - -use url::Url; - -fn main() { - let url = Url::parse("127.0.0.1").unwrap(); - let request = Request::new(Method::Post, url).unwrap(); - - let mut response = Multipart::from_request(request).unwrap() - .write_text("hello", "world") - .write_file("my_file", "my_text_data.txt") - .send().unwrap(); - - // Read response... -} -``` - -Server-side example using Hyper (`features = ["hyper", "server"]` or default): -```rust -use hyper::net::Fresh; -use hyper::server::{Server, Request, Response}; - -use multipart::server::Multipart; -use multipart::server::hyper::Switch; - -fn handle_regular<'a, 'k>(req: Request<'a, 'k>, res: Response<'a, Fresh>) { - // handle things here -} - -fn handle_multipart<'a, 'k>(mut multi: Multipart>, res: Response<'a, Fresh>) { - multi.foreach_entry(|entry| println!("Multipart entry: {:?}", entry)).unwrap(); -} - -fn main() { - Server::http("0.0.0.0:0").unwrap() - .handle(Switch::new(handle_regular, handle_multipart)) - .unwrap(); -} -``` +Supports several different HTTP crates. + +####[Documentation](http://cybergeek94.github.io/multipart/) + +###Integrations + +#####[Hyper](http://hyper.rs) +via the `hyper` feature (enabled by default). + +Client integration includes support for regular `hyper::client::Request` objects via `multipart::client::Multipart`, as well +as integration with the new `hyper::Client` API via `multipart::client::lazy::Mulitpart` (new in 0.5). + +Server integration for `hyper::server::Request` via `multipart::server::Multipart`. + +#####[Iron](http://ironframework.io) +via the `iron` feature (new in 0.5). + +Provides regular server-side integration with `iron::Request` via `multipart::server::Multipart`, +as well as a convenient `BeforeMiddleware` implementation in `multipart::server::iron::Intercept`. + +#####[tiny\_http](https://crates.io/crates/tiny_http/) +via the `tiny_http` feature (new in 0.5). + +Provides server-side integration with `tiny_http::Request` via `multipart::server::Multipart`. License ------- From eee1345295547b605357f5c7dca02db1f5bbdfd7 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sun, 13 Mar 2016 00:05:44 -0800 Subject: [PATCH 117/453] Change docs deploy process --- multipart/.travis.yml | 7 ++++++- multipart/deploy-docs.sh | 25 ++++++++++++++++++++++--- multipart/id_rsa.enc | Bin 0 -> 1686 bytes 3 files changed, 28 insertions(+), 4 deletions(-) create mode 100644 multipart/id_rsa.enc diff --git a/multipart/.travis.yml b/multipart/.travis.yml index 46f6f07cb..7674e82e7 100644 --- a/multipart/.travis.yml +++ b/multipart/.travis.yml @@ -8,10 +8,15 @@ os: - osx env: global: - - secure: UrIkXI/5PqRVn9GeCAtadkvIcnGY9+uMKfz50099M+ncf7/ZAwL5T6AjMmV9FqgvRShpW7px8Aj7uEGwbNgUGIuiYENWeoAi7LAh5OatUHBMVNFLnCLxHtnmj8G3RgNAql5wiEvvnaWvb+joi8zNJ4lJ5m6kkrAPxwx1Uj6uYmw= + - FEATURES="iron tiny_http" +script: + - cargo build -v --features ${FEATURES} + - cargo test -v --features ${FEATURES} + - cargo doc -v --no-deps --features ${FEATURES} after_success: - | test ${TRAVIS_PULL_REQUEST} == "false" && \ test ${TRAVIS_BRANCH} == "master" && \ test "${TRAVIS_BUILD_NUMBER}.1" == "${TRAVIS_JOB_NUMBER}" && \ + test -n ${TRAVIS_TAG} && \ bash deploy-docs.sh diff --git a/multipart/deploy-docs.sh b/multipart/deploy-docs.sh index 13ce69f83..aad6ef6d1 100644 --- a/multipart/deploy-docs.sh +++ b/multipart/deploy-docs.sh @@ -1,6 +1,25 @@ #!/bin/sh + +#Decrypt RSA key +mkdir -p ~/.ssh +openssl aes-256-cbc -K $encrypted_0a6446eb3ae3_key -iv $encrypted_0a6446eb3ae3_key -in id_rsa.enc -out ~/.ssh/id_rsa -d +chmod 600 ~/.ssh/id_rsa + cargo doc -v --no-deps -cp -R target/doc doc -curl http://www.rust-ci.org/artifacts/put?t=$RUSTCI_TOKEN | sh -rm -r doc + +git config user.name "multipart doc upload" +git config user.email "nobody@example.com" + +git branch -df gh-pages +git checkout --orphan gh-pages + +git clean -d -x -f -e target/doc + +cp -R target/doc . +rm -rf target + +git add -A + +git commit -qm "Documentation for ${TRAVIS_TAG}" +git push -f origin gh-pages diff --git a/multipart/id_rsa.enc b/multipart/id_rsa.enc new file mode 100644 index 0000000000000000000000000000000000000000..d8df97adba9c4e951f171dbe7e57e7ac17a64b56 GIT binary patch literal 1686 zcmV;H25I@*5aMsih+rN7(zl;@W)t+g+O#(%%!?0UTVZc0a^~&TKr;X#)es*Y4;%aR ztS1__rWsw^6)pc9d5HFpV!{>9cGqJ^cikr5TV_i-eEJqhvRhx|%xPO!9PeVf2;q>y z)b*!7#J@dy&w9L-gOXXY2-f&Jv; zl%p~IGsb(~WJsTX>3mX=q)012D4OHb(J{(L){*|S_kvR4>WJi`JTAVtOq-D83=rh^ z0(8lWLGFWYCK4^#@bj3EzxhS@)9nVy4CZytZ=09eGCPt}UsJ^rWG369Aa5Lb@I#{U zjGR9C>Rur{RmtvKF|Kp-n=PR>lhNj*nricBP6gS2!;QYF{ld&`6*^3~!%;J1`V9)l zoZAKEsdvbiv$Ja){f?P;yA_ReFW{g9TGv(o?p}yT{UBm@W3dS{@W=ydARsyT{T&p` zFBl8&7wW(Ftjy}z`zk9JK&g}G;MTyj=|R{fL~2X`i8MbtV`s&$YRVk=} zfVNn%HxH;W=GVV|wxz1RDi1mdeeyX?Z@=x|oqI@c#6TX(+RXVEQ?l?(Rgoz9Q z4GNu3dA)-rFM{m92nDYhQ|}CVs&k`5;C93lS;=cx_<~6E%4zqWm~4FMlHQhiZfvTx1{S3#@!im$XP;8FTOS{TC#PDLzUjK)>@@Vu zG~(ifMu%4}RMxB8M1zg#J_HG?<&qnN(z5u8ZnW=azkn2M^Lp9di%FM^1UUm_nlfto zDO)v9T5;~@AxHDwH?R4|F*o*8hfAh=EStwfc#RaZsz$KD6Ex}T$gbx3)$H%L8FD!q zxsifmTFc&b*{yT`l*O`h-(+LY-H0fh+O-8T4GQs!eyn2vzr-ylq_aIndVOv}&t>~I z^E`EiAqYq8|F-RU%+oeK-K3tj1vk_elg!Q?N;LY&B}l?T>j|{O zoHhd^#PLYrT0Cp=z!^pRhx+j#Nl{!5rQ9SdlKAFns3yw3O_q$bQt=ToTIr=aN%;1q za`nAK`GE+W4fj5hs-7^98TGZV>DL8Y*8P(5|Gn*;!_>{Xe!7!?7-jRbx<2nCjYB^* zvKN*R$)5>B<&lV1jP!|Sr$nMqoujpH*tWPIi^hEvj1YIleAcwQ*9{7`qhpl6*G!M? zfqA#!q`{WQbKP?KoVI)`z`)lHZaVy`~iLU z{%Evq26?epVKf zbLIG+y+QGKgWQ9_h&N=cnuX+sfukC%4d>w?G3W)#P}>csEC-M!iBK^Peo8f}kSJ1xkk;BhMf7`C!S`#dn=HG&MH&x*xu`ZJRDj@rb_Id z6aHabW9z#PQr|$==Wy4?9N<)k-<8XUKE}WKV-R(gzz_iSS3eKsd&Sh9G>h@;MvV4u zu!i??F^NCVkN!XbeyG**pNf~9d~33!2BIJs)QtneE~R?Pe_GEy!aCe?d3L?0{H>^a z7CCx}fGES!u-Ccv=KARX&QLrU@RdN^U*gv$4GNE!+(%gg>paqS2iC1iXVsm|OCO|x z;Q+`Ld6YHYRHAxYbD7PZZmSS91)4yiR5CwEzwz6Y9^F2T2IV;SkXRgKLk}_KW!yjE gN`$8Fh4&l4k65M8*<;&|Y>!%0EI-1$xoenYonzW)o&W#< literal 0 HcmV?d00001 From b744636bb5a0d2796af036b4c68324c1c2576266 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sun, 13 Mar 2016 00:12:12 -0800 Subject: [PATCH 118/453] Add git-reset to deploy-docs.sh --- multipart/deploy-docs.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/multipart/deploy-docs.sh b/multipart/deploy-docs.sh index aad6ef6d1..20e3aba37 100644 --- a/multipart/deploy-docs.sh +++ b/multipart/deploy-docs.sh @@ -14,6 +14,7 @@ git config user.email "nobody@example.com" git branch -df gh-pages git checkout --orphan gh-pages +git reset git clean -d -x -f -e target/doc cp -R target/doc . From a7d142b1ddb050f874d3afbbd74cf01b0a0d2a07 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sun, 13 Mar 2016 00:21:20 -0800 Subject: [PATCH 119/453] Fix Travis script --- multipart/.travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/multipart/.travis.yml b/multipart/.travis.yml index 7674e82e7..1117837b5 100644 --- a/multipart/.travis.yml +++ b/multipart/.travis.yml @@ -10,9 +10,9 @@ env: global: - FEATURES="iron tiny_http" script: - - cargo build -v --features ${FEATURES} - - cargo test -v --features ${FEATURES} - - cargo doc -v --no-deps --features ${FEATURES} + - cargo build -v --features "$FEATURES" + - cargo test -v --features "$FEATURES" + - cargo doc -v --no-deps --features "$FEATURES" after_success: - | test ${TRAVIS_PULL_REQUEST} == "false" && \ From f181efe7b8a707f65b964aced2622fd7c23e6f08 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sun, 13 Mar 2016 01:56:54 -0800 Subject: [PATCH 120/453] Bump version --- multipart/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 0ee75a67e..0632f5ae5 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "multipart" -version = "0.5.0-alpha.1" +version = "0.5.0" authors = ["Austin Bonander "] From 8663e21c3fe79c4929564f8139d7bfd6a81d40ed Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sun, 13 Mar 2016 03:03:16 -0700 Subject: [PATCH 121/453] Remove test for branch == master in travis --- multipart/.travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/multipart/.travis.yml b/multipart/.travis.yml index 1117837b5..4d3a1c5f9 100644 --- a/multipart/.travis.yml +++ b/multipart/.travis.yml @@ -16,7 +16,6 @@ script: after_success: - | test ${TRAVIS_PULL_REQUEST} == "false" && \ - test ${TRAVIS_BRANCH} == "master" && \ test "${TRAVIS_BUILD_NUMBER}.1" == "${TRAVIS_JOB_NUMBER}" && \ test -n ${TRAVIS_TAG} && \ bash deploy-docs.sh From 1c1a81e41ec7ad9279aed5294aff69cbfbf46aeb Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sun, 13 Mar 2016 03:17:24 -0700 Subject: [PATCH 122/453] Fix deploy-docs --- multipart/deploy-docs.sh | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/multipart/deploy-docs.sh b/multipart/deploy-docs.sh index 20e3aba37..1195e4598 100644 --- a/multipart/deploy-docs.sh +++ b/multipart/deploy-docs.sh @@ -3,19 +3,16 @@ #Decrypt RSA key mkdir -p ~/.ssh -openssl aes-256-cbc -K $encrypted_0a6446eb3ae3_key -iv $encrypted_0a6446eb3ae3_key -in id_rsa.enc -out ~/.ssh/id_rsa -d +openssl aes-256-cbc -K $encrypted_9e7566e23d77_key -iv $encrypted_9e7566e23d77_iv -in id_rsa.enc -out ~/.ssh/id_rsa -d chmod 600 ~/.ssh/id_rsa -cargo doc -v --no-deps - git config user.name "multipart doc upload" git config user.email "nobody@example.com" -git branch -df gh-pages git checkout --orphan gh-pages git reset -git clean -d -x -f -e target/doc +git clean -d -x -f -e target cp -R target/doc . rm -rf target From ea0293146176c4114cb5c5e34a25cacd32fc3dc1 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sun, 13 Mar 2016 03:24:26 -0700 Subject: [PATCH 123/453] Update RSA key --- multipart/deploy-docs.sh | 2 +- multipart/id_rsa.enc | Bin 1686 -> 1688 bytes 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/deploy-docs.sh b/multipart/deploy-docs.sh index 1195e4598..5739af31b 100644 --- a/multipart/deploy-docs.sh +++ b/multipart/deploy-docs.sh @@ -3,7 +3,7 @@ #Decrypt RSA key mkdir -p ~/.ssh -openssl aes-256-cbc -K $encrypted_9e7566e23d77_key -iv $encrypted_9e7566e23d77_iv -in id_rsa.enc -out ~/.ssh/id_rsa -d +openssl aes-256-cbc -K $encrypted_9e7566e23d77_key -iv $encrypted_9e7566e23d77_iv -in id_rsa.enc -out id_rsa -d chmod 600 ~/.ssh/id_rsa git config user.name "multipart doc upload" diff --git a/multipart/id_rsa.enc b/multipart/id_rsa.enc index d8df97adba9c4e951f171dbe7e57e7ac17a64b56..ab0bcd3e286e269a7262602c1d238ff27037c22e 100644 GIT binary patch literal 1688 zcmV;J250$+!XFZh`Bb~tL36XHd8oO2a^-d%ra0Q0vtJ%nN2?-fe6x`_g!nnoy@VKQP3ti76c(5%1DYdyWr zCydd1h+P4_Q(JgctF#G|F-V)-tnG=fvCti^@3OP&x5V*E2jqVJ;C|i0a{Id)lf}Mk zDReBc9~>*pR?s?xTat@|O4Ao%EX-M{6oeI9zD-5k-?uR}CfPC@$v4&qkKWN+hLvVSX?$TccsNS zkJk6cODLinwl)x1uexnvl#MyCd&%W37;2Xsp+6O-0NbpH&?Na%iSxND$fQwSqV8MG zMj9G5Mri4q^lw*DMv#2jah~&GxI3oR;Jn7|OoCF4eE8U~B@GH*X&)|rjSC`;QCPiTK8F( zqyP;HGwN*>$_Uq4>nW!8>5F zQ;g|#&}7>g=86pM=q+pHsXAJRw|PG?v}U4t12BzfZCu?*-+#$?W*dHLd{UBAG1V zubpaPChnge>)+Okp@Zf-&V*TvM1rsu7VyhQdGszMDt+fmdIVA*z5Kl>*EmC`rjQoQ zu-b-t+YYjlgLAP-gZ}VrbT+&b_M35aq=GknpqS#xDISJXkCxt# z1d9AnZ|g*H%d9<90i?bw(s$RpRZW~Wwsa)E;r)W&<0w{|jkG>W&xJcfYf|Zb8 z;EQOT`SqRfUeugL*6-L3$u;x+0qF)=$o>C={8Dc!82xCWwLb6jE3{oxU10ZJWdvJL zUhnUgH{Y|JtrNmh^~E5M!hEDx*5wT0bL(K9Q9Gpq8!($VlP^;z1JP5%{aDy^0g{p= zV(|nhQoBAbZQJ|+SE2K0Qu3N!w&X7~s-D)24vg4SnTj*b)uW8uF{L~>Q0-a*#NSJO z?+?HrX-03yVIK8nnIus2v)OJ*X(Z!j7#!EAm{*7lhNRr|t(v3D0Pr6uik)^YRW*D* zA1ibd>jvn!XRA1Iqg?le$;Z4@&vGr;ttp@8teCc#7f0-safnf=yV16j)K;A6Rw1+4 zChk)(B_0O_Bkhv}SWpfn%>T-WLFZ=Klfk}HxDO`z%XzJ;c?TS>7@2U zusd#osF{bB)Sdy?t3J`ArT-|@;ArG~_hn^t_QPY<)D2o5hQZnQbJik)mbV4w9cGqJ^cikr5TV_i-eEJqhvRhx|%xPO!9PeVf2;q>y z)b*!7#J@dy&w9L-gOXXY2-f&Jv; zl%p~IGsb(~WJsTX>3mX=q)012D4OHb(J{(L){*|S_kvR4>WJi`JTAVtOq-D83=rh^ z0(8lWLGFWYCK4^#@bj3EzxhS@)9nVy4CZytZ=09eGCPt}UsJ^rWG369Aa5Lb@I#{U zjGR9C>Rur{RmtvKF|Kp-n=PR>lhNj*nricBP6gS2!;QYF{ld&`6*^3~!%;J1`V9)l zoZAKEsdvbiv$Ja){f?P;yA_ReFW{g9TGv(o?p}yT{UBm@W3dS{@W=ydARsyT{T&p` zFBl8&7wW(Ftjy}z`zk9JK&g}G;MTyj=|R{fL~2X`i8MbtV`s&$YRVk=} zfVNn%HxH;W=GVV|wxz1RDi1mdeeyX?Z@=x|oqI@c#6TX(+RXVEQ?l?(Rgoz9Q z4GNu3dA)-rFM{m92nDYhQ|}CVs&k`5;C93lS;=cx_<~6E%4zqWm~4FMlHQhiZfvTx1{S3#@!im$XP;8FTOS{TC#PDLzUjK)>@@Vu zG~(ifMu%4}RMxB8M1zg#J_HG?<&qnN(z5u8ZnW=azkn2M^Lp9di%FM^1UUm_nlfto zDO)v9T5;~@AxHDwH?R4|F*o*8hfAh=EStwfc#RaZsz$KD6Ex}T$gbx3)$H%L8FD!q zxsifmTFc&b*{yT`l*O`h-(+LY-H0fh+O-8T4GQs!eyn2vzr-ylq_aIndVOv}&t>~I z^E`EiAqYq8|F-RU%+oeK-K3tj1vk_elg!Q?N;LY&B}l?T>j|{O zoHhd^#PLYrT0Cp=z!^pRhx+j#Nl{!5rQ9SdlKAFns3yw3O_q$bQt=ToTIr=aN%;1q za`nAK`GE+W4fj5hs-7^98TGZV>DL8Y*8P(5|Gn*;!_>{Xe!7!?7-jRbx<2nCjYB^* zvKN*R$)5>B<&lV1jP!|Sr$nMqoujpH*tWPIi^hEvj1YIleAcwQ*9{7`qhpl6*G!M? zfqA#!q`{WQbKP?KoVI)`z`)lHZaVy`~iLU z{%Evq26?epVKf zbLIG+y+QGKgWQ9_h&N=cnuX+sfukC%4d>w?G3W)#P}>csEC-M!iBK^Peo8f}kSJ1xkk;BhMf7`C!S`#dn=HG&MH&x*xu`ZJRDj@rb_Id z6aHabW9z#PQr|$==Wy4?9N<)k-<8XUKE}WKV-R(gzz_iSS3eKsd&Sh9G>h@;MvV4u zu!i??F^NCVkN!XbeyG**pNf~9d~33!2BIJs)QtneE~R?Pe_GEy!aCe?d3L?0{H>^a z7CCx}fGES!u-Ccv=KARX&QLrU@RdN^U*gv$4GNE!+(%gg>paqS2iC1iXVsm|OCO|x z;Q+`Ld6YHYRHAxYbD7PZZmSS91)4yiR5CwEzwz6Y9^F2T2IV;SkXRgKLk}_KW!yjE gN`$8Fh4&l4k65M8*<;&|Y>!%0EI-1$xoenYonzW)o&W#< From 60644f64a23103d76c4c342d8a8ba6fe833559b6 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sun, 13 Mar 2016 03:45:42 -0700 Subject: [PATCH 124/453] Update RSA key --- multipart/deploy-docs.sh | 2 +- multipart/id_rsa.enc | Bin 1688 -> 3248 bytes 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/deploy-docs.sh b/multipart/deploy-docs.sh index 5739af31b..6ab0746ac 100644 --- a/multipart/deploy-docs.sh +++ b/multipart/deploy-docs.sh @@ -3,7 +3,7 @@ #Decrypt RSA key mkdir -p ~/.ssh -openssl aes-256-cbc -K $encrypted_9e7566e23d77_key -iv $encrypted_9e7566e23d77_iv -in id_rsa.enc -out id_rsa -d +openssl aes-256-cbc -K $encrypted_67079d13adee_key -iv $encrypted_67079d13adee_iv -in id_rsa.enc -out id_rsa -d chmod 600 ~/.ssh/id_rsa git config user.name "multipart doc upload" diff --git a/multipart/id_rsa.enc b/multipart/id_rsa.enc index ab0bcd3e286e269a7262602c1d238ff27037c22e..57b91eb9a27d5607d18e041ad2be203381dfbb70 100644 GIT binary patch literal 3248 zcmV;h3{Ug<(--whIQygTHqySZ8fQ*C9@Bayb&6J2%?dIGbz8szkt2>AShqfiBd~G= zz9=j^ALCvX$&A8OVV79?8(AgCO)FRE~>13?Aqc^d_*yXa+h5w-k6Tj#K@r{r1z zyYSjoi#}n1mvmjo;vm)OFRTFDK?Jn@s#!5NxdW#TpZE2_v*yoMmHc}8fWi|&ITZ*| zi#rd44IJaA zwMt}^1Oo`@jU|}v0F*ODAl>_H=U%u_21 z!rQK{h_G1)pQ?yQo{aJYtQP)uYHOjyRNFIpi*ymuomw=^h=&Z4ux9z-vXpi)C$T}Y zWH3zu^OfnskAIG0z5BzVDmq_&O50c_A#0+KWF*6c1_UFuODJQ%_wAiiBvHPrAhD!* za^r&>z~qT3W_OQ0+Hr;IPoL%+rLNT#Ee~Ww^WX0pNo!!RebWGI-XX%1-4Y* zdX%_1;MAvBT#+!$)5zuCU*qE?Bv9%r8(KKBT$Wy0;xjb$YtCyko$8>#?Fg=ihjJ?0 zAq3#~AK2puZo|w*ql~_Nzf%>1v;04{!f1Di%MCCl8o(renB% z{N$4p0ewM?j~UJB`m1uHh04U~4z9vu<7F?@Z^?aAP82iYNcx}ZC+2zhR%9_%6C1Hg z;4Sm^1$^qvQXv!k&L{>c$Ecv0yq1}xBq7ej5oIOY!@e!7U)284))uV+5*km*;BaS zhiRcMX|fTzAZ?74@fpxrbnF=eQ?>{8@&mcpEEFO|(bT%!xGsF1@^dGF*_0;*0;UxB zA742eKBp=w*L^=gR4;fQDtX)ktKYJ(kjidJFxv5hG03^>K2LH4pl&WxqsN^8DC$Z# z5YdwyK|3p?P}6Ogl&e@#1?#>{rd9VgEud7YpI*J^(K>G#org@qCPa9TI{Gz)&xN{LL~F;eQE^f%E!HU9vPz< zk@^ZHnVDV1M>O(DSmYUQ?l#gi7Tx=dveH{n^n3=zJ<%bb3|8cWczPpG=>XrUH3?hv z-jcK@QjjA^=JT0!Za!bxBMbTfbI?u)e7@hpijazEFQ6O3Vc@Oc&DL(YlSv#3SpO$A zC9=I#W8sfN!8fDK`AR&zJ6ba%)^PjPxHu!WRZ9805N{TJvn%tzcu;!J1&{Ql+I`J# z7LhyyFDuGZzj{XAjx1lY!oSPlRl*Eg17A^DIQS+=hM3K{U@B79`<4@Opl$Qq$_NYH zyyvchxvDoB!ONYkAO-WmR-oi)gYuc*iW#NvGMalp_+LG8Tac^IqhND;W}y}(GK&_N z%piA5E(XgPmBfAOvzA7`@R?=6t;%kB>JwC9#IGRn0!x-Vx+ z0u;{krJIP{i;OjeaVoCboOTUd78e*by(V@tgg;QzyBWqbC$l|9xV!=O>a&hS^7C&# z#^T2f;w!J4xX(b0Q4&HUX5hHK+`Ql@|Mfc{Z#e5WOJNHjqd>yoeCxzZxW=)5l7vps zkpcJe=5dunK?%sO6wF_%8^4205px&mPV|8onaE!?eY2oJo!&B-rjJTFrh_AZPu>>( z3%+>3Ee`AqQDo%|S834ItCeo61XfMuP-4rz(Fi#UC&2BL!zi-sU}6VCoY5!*tXbQ% zdv%p0F$9fMA|)f{t<_*Z9dGh2fWf5)+tl6bw6zm?F3-rDvqdfk(|JyNZ#08`loeme*N}y*2Jf>2(>W~*512Ks zWk^>?;dj7ikvldi6p;2E4I&;O%L9ClwXfnig#niQ@ZRcs5jSZT*b-2U^6C$Oot}RY zj0);FQX>MF8~rkgQM^9AUrDLqq|aY`kFytHjpO9O6#{A5jaQ|%=K|L9A;vCHLpPcI zE~{U|3Wq<<+DxJxQa(cL7Eznc$~3~-Nf#E5D5aOyfRLpRrKD+Qk11;<>-N?DrBmT8(0)MjqAvN+<5J(+gr) zRPu7u{Oq=aVgUe%bH^~~!!^|Qvi4iqk=?7GMcw>eaYs;x zQc@l$YHk&yu8#7LZ|>fAs-Ub((aOMXulr_X3FY#|9pZFYjLJ+ucp3slv$UFz zL}!B3PZ#w5~37I40cA&rwve9|RF{g{Ni!b5~olAqxWruUrRmW8&Upn8^5EkYRss)OQEtwT6|wAmvlxIg!rFziJuKFV>tOMSO?LZ@!VtNLO4P zo_&ixn09NEPO5|h^#q-l(oDJ6k;cxoGXJ&(-oXidKs>P-krr59)ft1CRn?`G;Io#je5PyP5X|hYxUCXi zYT z+lWhyNt+)@+r0#6?`%4~tl5!_m22$Z!qy6yoVBG~VM6c}+~!S2WPa4YHB_*Za%adV zpb9<+b;ZdP0%-%u_-)@StnA3(7YtK9Fi?GDDHiv2sLG?S{}roE0P{dmlIjWUJ0=XE zSZ1hI$+PSFkF7(@2J+RhOmd)Y#y1FQqDq(B&yx2T^@R9?k{Bq?^Wky8m?jl$Sjj9A zO=nsaARhT=2>o{z7S83o_b>oYy%_Q?JwEpm9Di)J%r7&dHb)er1`O)iuYAC0!C!E1 zC>5d=8o84N+v;a|uH&0d>k~MBX1|mAkfJT3HzmP%6&XpO%JPY}F(yk#!2?i?0ps3t z^+Pfy<#!M1Y)>^pOlrCJC^YFX_+>$ ibOz?PRC&r%f0n>{-?UQNIbfGrs=8oKVl)CV^q5Y13usXQ literal 1688 zcmV;J250$+!XFZh`Bb~tL36XHd8oO2a^-d%ra0Q0vtJ%nN2?-fe6x`_g!nnoy@VKQP3ti76c(5%1DYdyWr zCydd1h+P4_Q(JgctF#G|F-V)-tnG=fvCti^@3OP&x5V*E2jqVJ;C|i0a{Id)lf}Mk zDReBc9~>*pR?s?xTat@|O4Ao%EX-M{6oeI9zD-5k-?uR}CfPC@$v4&qkKWN+hLvVSX?$TccsNS zkJk6cODLinwl)x1uexnvl#MyCd&%W37;2Xsp+6O-0NbpH&?Na%iSxND$fQwSqV8MG zMj9G5Mri4q^lw*DMv#2jah~&GxI3oR;Jn7|OoCF4eE8U~B@GH*X&)|rjSC`;QCPiTK8F( zqyP;HGwN*>$_Uq4>nW!8>5F zQ;g|#&}7>g=86pM=q+pHsXAJRw|PG?v}U4t12BzfZCu?*-+#$?W*dHLd{UBAG1V zubpaPChnge>)+Okp@Zf-&V*TvM1rsu7VyhQdGszMDt+fmdIVA*z5Kl>*EmC`rjQoQ zu-b-t+YYjlgLAP-gZ}VrbT+&b_M35aq=GknpqS#xDISJXkCxt# z1d9AnZ|g*H%d9<90i?bw(s$RpRZW~Wwsa)E;r)W&<0w{|jkG>W&xJcfYf|Zb8 z;EQOT`SqRfUeugL*6-L3$u;x+0qF)=$o>C={8Dc!82xCWwLb6jE3{oxU10ZJWdvJL zUhnUgH{Y|JtrNmh^~E5M!hEDx*5wT0bL(K9Q9Gpq8!($VlP^;z1JP5%{aDy^0g{p= zV(|nhQoBAbZQJ|+SE2K0Qu3N!w&X7~s-D)24vg4SnTj*b)uW8uF{L~>Q0-a*#NSJO z?+?HrX-03yVIK8nnIus2v)OJ*X(Z!j7#!EAm{*7lhNRr|t(v3D0Pr6uik)^YRW*D* zA1ibd>jvn!XRA1Iqg?le$;Z4@&vGr;ttp@8teCc#7f0-safnf=yV16j)K;A6Rw1+4 zChk)(B_0O_Bkhv}SWpfn%>T-WLFZ=Klfk}HxDO`z%XzJ;c?TS>7@2U zusd#osF{bB)Sdy?t3J`ArT-|@;ArG~_hn^t_QPY<)D2o5hQZnQbJik)mbV4w Date: Sun, 13 Mar 2016 03:56:33 -0700 Subject: [PATCH 125/453] Fix deploy-docs (again) --- multipart/deploy-docs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/deploy-docs.sh b/multipart/deploy-docs.sh index 6ab0746ac..5172362dc 100644 --- a/multipart/deploy-docs.sh +++ b/multipart/deploy-docs.sh @@ -3,7 +3,7 @@ #Decrypt RSA key mkdir -p ~/.ssh -openssl aes-256-cbc -K $encrypted_67079d13adee_key -iv $encrypted_67079d13adee_iv -in id_rsa.enc -out id_rsa -d +openssl aes-256-cbc -K $encrypted_67079d13adee_key -iv $encrypted_67079d13adee_iv -in id_rsa.enc -out ~/.ssh/id_rsa -d chmod 600 ~/.ssh/id_rsa git config user.name "multipart doc upload" From af34fc26038d5fe76bdd6e5dac5555af9979ee4b Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sun, 13 Mar 2016 04:04:08 -0700 Subject: [PATCH 126/453] Fix deploy-docs (again) --- multipart/deploy-docs.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/multipart/deploy-docs.sh b/multipart/deploy-docs.sh index 5172362dc..638c1530f 100644 --- a/multipart/deploy-docs.sh +++ b/multipart/deploy-docs.sh @@ -20,4 +20,5 @@ rm -rf target git add -A git commit -qm "Documentation for ${TRAVIS_TAG}" +git remote set-url origin git@github.com:cybergeek94/multipart.git git push -f origin gh-pages From a5326f2a9259a1cf6372075476af6ea22bb4f1dc Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sun, 13 Mar 2016 04:38:23 -0700 Subject: [PATCH 127/453] Fix internal links in docs, update documentation links --- multipart/Cargo.toml | 4 ++-- multipart/README.md | 10 +++++----- multipart/src/client/hyper.rs | 6 ++---- multipart/src/client/lazy.rs | 2 +- multipart/src/lib.rs | 6 ++++++ 5 files changed, 16 insertions(+), 12 deletions(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 0632f5ae5..82eabad58 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -7,12 +7,12 @@ authors = ["Austin Bonander "] description = "A backend-agnostic extension for HTTP libraries that provides support for POST multipart/form-data requests on for both client and server." -documentation = "http://rust-ci.org/cybergeek94/multipart/doc/multipart/" - keywords = ["form-data", "hyper", "iron", "http", "upload"] repository = "http://github.com/cybergeek94/multipart" +documentation = "http://cybergeek94.github.io/multipart/doc/multipart/index.html" + license = "MIT OR Apache-2.0" [dependencies] diff --git a/multipart/README.md b/multipart/README.md index 33cd8ff24..992cedf7c 100644 --- a/multipart/README.md +++ b/multipart/README.md @@ -5,11 +5,11 @@ Client- and server-side abstractions for HTTP file uploads (POST requests with Supports several different HTTP crates. -####[Documentation](http://cybergeek94.github.io/multipart/) +###[Documentation](http://cybergeek94.github.io/multipart/doc/multipart/index.html) -###Integrations +##Integrations -#####[Hyper](http://hyper.rs) +####[Hyper](http://hyper.rs) via the `hyper` feature (enabled by default). Client integration includes support for regular `hyper::client::Request` objects via `multipart::client::Multipart`, as well @@ -17,13 +17,13 @@ as integration with the new `hyper::Client` API via `multipart::client::lazy::Mu Server integration for `hyper::server::Request` via `multipart::server::Multipart`. -#####[Iron](http://ironframework.io) +####[Iron](http://ironframework.io) via the `iron` feature (new in 0.5). Provides regular server-side integration with `iron::Request` via `multipart::server::Multipart`, as well as a convenient `BeforeMiddleware` implementation in `multipart::server::iron::Intercept`. -#####[tiny\_http](https://crates.io/crates/tiny_http/) +####[tiny\_http](https://crates.io/crates/tiny_http/) via the `tiny_http` feature (new in 0.5). Provides server-side integration with `tiny_http::Request` via `multipart::server::Multipart`. diff --git a/multipart/src/client/hyper.rs b/multipart/src/client/hyper.rs index 2d6e4e34f..487191676 100644 --- a/multipart/src/client/hyper.rs +++ b/multipart/src/client/hyper.rs @@ -9,11 +9,9 @@ //! //! Contains `impl HttpRequest for Request` and `impl HttpStream for Request`. //! -//! Also see: [`lazy::Multipart::client_request()`][lazy-multi-creq] -//! and [`lazy::Multipart::client_request_mut()`][lazy-multi-creq-mut] +//! Also see: [`lazy::Multipart::client_request()`](../lazy/struct.Multipart.html#method.client_request) +//! and [`lazy::Multipart::client_request_mut()`](../lazy/struct.Multipart.html#method.client_request_mut) //! (adaptors for `hyper::client::RequestBuilder`). -//! [lazy-multipart-creq]: ../lazy/struct.Multipart.html#method.client_request -//! [lazy-mutlipart-creq-mut]: ../lazy/struct.Multipart.html#method.client_request_mut use hyper::client::request::Request; use hyper::client::response::Response; use hyper::header::{ContentType, ContentLength}; diff --git a/multipart/src/client/lazy.rs b/multipart/src/client/lazy.rs index 90df0b849..ae88db38c 100644 --- a/multipart/src/client/lazy.rs +++ b/multipart/src/client/lazy.rs @@ -238,7 +238,7 @@ struct Stream<'n, 'd> { stream: Box, } -/// The result of [`Multipart::prepare()`](../Multipart.html#method.prepare) or +/// The result of [`Multipart::prepare()`](struct.Multipart.html#method.prepare) or /// `Multipart::prepare_threshold()`. Implements `Read`, contains the entire request body. pub struct PreparedFields<'d> { // NOTE: the order of these fields have been reversed so fields can be popped one-by-one from diff --git a/multipart/src/lib.rs b/multipart/src/lib.rs index 44b7a233f..0d39bcd83 100644 --- a/multipart/src/lib.rs +++ b/multipart/src/lib.rs @@ -16,6 +16,12 @@ //! //! * `hyper` (default): Enable integration with the [Hyper](https:://github.com/hyperium/hyper) HTTP library //! for client and/or server depending on which other feature flags are set. +//! +//! * `iron`: Enable integration with the [Iron](http://ironframework.io) web application +//! framework. See the [`server::iron`](server/iron/index.html) module for more information. +//! +//! * `tiny_http`: Enable integration with the [`tiny_http`](https://github.com/frewsxcv/tiny-http) +//! crate. See the [`server::tiny_http`](server/tiny_http/index.html) module for more information. #![warn(missing_docs)] #[macro_use] extern crate log; extern crate env_logger; From d9afbe602985120d111e699f834b2dbe65462f67 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Thu, 17 Mar 2016 21:23:59 -0700 Subject: [PATCH 128/453] Fix breakages due to mime_guess --- multipart/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 82eabad58..aaabe47be 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "multipart" -version = "0.5.0" +version = "0.5.1" authors = ["Austin Bonander "] @@ -19,7 +19,7 @@ license = "MIT OR Apache-2.0" env_logger = "0.3" log = "0.3" mime = "0.1" -mime_guess = "1.4" +mime_guess = "=1.5" rand = "0.3" tempdir = "0.3" From 526c4f743430f9e84782c0545b4f2397b63bddb2 Mon Sep 17 00:00:00 2001 From: White-Oak Date: Sat, 19 Mar 2016 06:39:24 +0300 Subject: [PATCH 129/453] Added sample for tiny_http feature --- multipart/samples/tiny_http/.gitignore | 1 + multipart/samples/tiny_http/Cargo.lock | 314 ++++++++++++++++++++++++ multipart/samples/tiny_http/Cargo.toml | 12 + multipart/samples/tiny_http/src/main.rs | 44 ++++ 4 files changed, 371 insertions(+) create mode 100644 multipart/samples/tiny_http/.gitignore create mode 100644 multipart/samples/tiny_http/Cargo.lock create mode 100644 multipart/samples/tiny_http/Cargo.toml create mode 100644 multipart/samples/tiny_http/src/main.rs diff --git a/multipart/samples/tiny_http/.gitignore b/multipart/samples/tiny_http/.gitignore new file mode 100644 index 000000000..2f7896d1d --- /dev/null +++ b/multipart/samples/tiny_http/.gitignore @@ -0,0 +1 @@ +target/ diff --git a/multipart/samples/tiny_http/Cargo.lock b/multipart/samples/tiny_http/Cargo.lock new file mode 100644 index 000000000..a06ff8547 --- /dev/null +++ b/multipart/samples/tiny_http/Cargo.lock @@ -0,0 +1,314 @@ +[root] +name = "tiny_http" +version = "0.1.0" +dependencies = [ + "multipart 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "tiny_http 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "aho-corasick" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "memchr 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "ascii" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "buf_redux" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "chrono" +version = "0.2.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "num 0.1.31 (registry+https://github.com/rust-lang/crates.io-index)", + "time 0.1.34 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "chunked_transfer" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "encoding" +version = "0.2.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "encoding-index-japanese 1.20141219.5 (registry+https://github.com/rust-lang/crates.io-index)", + "encoding-index-korean 1.20141219.5 (registry+https://github.com/rust-lang/crates.io-index)", + "encoding-index-simpchinese 1.20141219.5 (registry+https://github.com/rust-lang/crates.io-index)", + "encoding-index-singlebyte 1.20141219.5 (registry+https://github.com/rust-lang/crates.io-index)", + "encoding-index-tradchinese 1.20141219.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "encoding-index-japanese" +version = "1.20141219.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "encoding_index_tests 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "encoding-index-korean" +version = "1.20141219.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "encoding_index_tests 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "encoding-index-simpchinese" +version = "1.20141219.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "encoding_index_tests 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "encoding-index-singlebyte" +version = "1.20141219.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "encoding_index_tests 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "encoding-index-tradchinese" +version = "1.20141219.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "encoding_index_tests 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "encoding_index_tests" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "env_logger" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "log 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 0.1.58 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "kernel32-sys" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "libc" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "log" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "matches" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "memchr" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "mime" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "log 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 0.6.15 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "mime_guess" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "mime 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "phf 0.7.13 (registry+https://github.com/rust-lang/crates.io-index)", + "phf_codegen 0.7.13 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "multipart" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "buf_redux 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "env_logger 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", + "mime 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "mime_guess 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", + "tempdir 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "tiny_http 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "num" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "phf" +version = "0.7.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "phf_shared 0.7.13 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "phf_codegen" +version = "0.7.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "phf_generator 0.7.13 (registry+https://github.com/rust-lang/crates.io-index)", + "phf_shared 0.7.13 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "phf_generator" +version = "0.7.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "phf_shared 0.7.13 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "phf_shared" +version = "0.7.13" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "rand" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "regex" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "aho-corasick 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", + "regex-syntax 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "utf8-ranges 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "regex-syntax" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "rustc-serialize" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "serde" +version = "0.6.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "num 0.1.31 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "tempdir" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "time" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "kernel32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "tiny_http" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "ascii 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)", + "chrono 0.2.20 (registry+https://github.com/rust-lang/crates.io-index)", + "chunked_transfer 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "encoding 0.2.32 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc-serialize 0.3.18 (registry+https://github.com/rust-lang/crates.io-index)", + "url 0.2.38 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "url" +version = "0.2.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "matches 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc-serialize 0.3.18 (registry+https://github.com/rust-lang/crates.io-index)", + "uuid 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "utf8-ranges" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "uuid" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc-serialize 0.3.18 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "winapi" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "winapi-build" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + diff --git a/multipart/samples/tiny_http/Cargo.toml b/multipart/samples/tiny_http/Cargo.toml new file mode 100644 index 000000000..fe5bfddfc --- /dev/null +++ b/multipart/samples/tiny_http/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "tiny_http" +version = "0.1.0" +authors = ["White-Oak "] + +[dependencies] +tiny_http = "0.5" + +[dependencies.multipart] +version = "0.5.1" +default-features = false +features = ["server", "tiny_http"] diff --git a/multipart/samples/tiny_http/src/main.rs b/multipart/samples/tiny_http/src/main.rs new file mode 100644 index 000000000..b94bab951 --- /dev/null +++ b/multipart/samples/tiny_http/src/main.rs @@ -0,0 +1,44 @@ +extern crate tiny_http; +extern crate multipart; + +use multipart::server::{Multipart, Entries, SaveResult}; +fn main() { + let server = tiny_http::Server::http("localhost:80").unwrap(); + loop { + // blocks until the next request is received + let mut request = server.recv().unwrap(); + + let mut multipart = Multipart::from_request(&mut request).unwrap(); + match multipart.save_all() { + SaveResult::Full(entries) => process_entries(entries).unwrap(), + SaveResult::Partial(entries, error) => { + process_entries(entries).unwrap(); + panic!("{:?}", error) + }, + SaveResult::Error(error) => panic!("{:?}", error) + } + } +} + +use std::io::prelude::*; +use std::fs::File; +use std::io::Error; +fn process_entries(entries: Entries) -> Result<(), Error> { + for (name, field) in entries.fields { + println!(r#"Field "{}": "{}""#, name, field); + } + + for (name, savedfile) in entries.files { + let filename = match savedfile.filename { + Some(s) => s, + None => "None".into() + }; + let mut f = try!(File::open(savedfile.path)); + + let mut s = String::new(); + try!(f.read_to_string(&mut s)); + println!(r#"Field "{}" is file "{}":"#, name, filename); + println!("{}", s); + } + Ok(()) +} From 015d2623b33114f7f2cb3b0afa579db5e7228904 Mon Sep 17 00:00:00 2001 From: White-Oak Date: Sat, 19 Mar 2016 06:51:49 +0300 Subject: [PATCH 130/453] Commented the sample for tiny_http --- multipart/samples/tiny_http/src/main.rs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/multipart/samples/tiny_http/src/main.rs b/multipart/samples/tiny_http/src/main.rs index b94bab951..acaf9f5f9 100644 --- a/multipart/samples/tiny_http/src/main.rs +++ b/multipart/samples/tiny_http/src/main.rs @@ -3,12 +3,15 @@ extern crate multipart; use multipart::server::{Multipart, Entries, SaveResult}; fn main() { + // Starting a server on `localhost:80` let server = tiny_http::Server::http("localhost:80").unwrap(); loop { - // blocks until the next request is received + // This blocks until the next request is received let mut request = server.recv().unwrap(); + // Getting a multipart reader wrapper let mut multipart = Multipart::from_request(&mut request).unwrap(); + // Fetching all data and processing it match multipart.save_all() { SaveResult::Full(entries) => process_entries(entries).unwrap(), SaveResult::Partial(entries, error) => { @@ -33,12 +36,12 @@ fn process_entries(entries: Entries) -> Result<(), Error> { Some(s) => s, None => "None".into() }; - let mut f = try!(File::open(savedfile.path)); + let mut file = try!(File::open(savedfile.path)); + let mut contents = String::new(); + try!(file.read_to_string(&mut contents)); - let mut s = String::new(); - try!(f.read_to_string(&mut s)); println!(r#"Field "{}" is file "{}":"#, name, filename); - println!("{}", s); + println!("{}", contents); } Ok(()) } From a6bf929245814896b04b454140d0a2515af1a51b Mon Sep 17 00:00:00 2001 From: White-Oak Date: Sat, 19 Mar 2016 07:01:11 +0300 Subject: [PATCH 131/453] Removed unused inports --- multipart/samples/tiny_http/src/main.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/multipart/samples/tiny_http/src/main.rs b/multipart/samples/tiny_http/src/main.rs index acaf9f5f9..e1e64ba9d 100644 --- a/multipart/samples/tiny_http/src/main.rs +++ b/multipart/samples/tiny_http/src/main.rs @@ -23,9 +23,8 @@ fn main() { } } -use std::io::prelude::*; use std::fs::File; -use std::io::Error; +use std::io::{Error, Read}; fn process_entries(entries: Entries) -> Result<(), Error> { for (name, field) in entries.fields { println!(r#"Field "{}": "{}""#, name, field); From 634c11d71b7ed8d4ad4bba8e118ae3d7a9c23f96 Mon Sep 17 00:00:00 2001 From: White-Oak Date: Sat, 19 Mar 2016 07:11:46 +0300 Subject: [PATCH 132/453] Added a response for tiny_http sample --- multipart/samples/tiny_http/src/main.rs | 30 ++++++++++++++++--------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/multipart/samples/tiny_http/src/main.rs b/multipart/samples/tiny_http/src/main.rs index e1e64ba9d..1489f3992 100644 --- a/multipart/samples/tiny_http/src/main.rs +++ b/multipart/samples/tiny_http/src/main.rs @@ -2,27 +2,35 @@ extern crate tiny_http; extern crate multipart; use multipart::server::{Multipart, Entries, SaveResult}; +use tiny_http::{Response, StatusCode, Request}; fn main() { // Starting a server on `localhost:80` let server = tiny_http::Server::http("localhost:80").unwrap(); loop { // This blocks until the next request is received let mut request = server.recv().unwrap(); + process_multipart(&mut request).unwrap(); - // Getting a multipart reader wrapper - let mut multipart = Multipart::from_request(&mut request).unwrap(); - // Fetching all data and processing it - match multipart.save_all() { - SaveResult::Full(entries) => process_entries(entries).unwrap(), - SaveResult::Partial(entries, error) => { - process_entries(entries).unwrap(); - panic!("{:?}", error) - }, - SaveResult::Error(error) => panic!("{:?}", error) - } + // Answering with an HTTP OK and a string + let response_string = "Multipart data is received!".as_bytes(); + let response = Response::new(StatusCode(200), vec![], response_string, Some(response_string.len()), None); + request.respond(response).unwrap(); } } +fn process_multipart(request: &mut Request) -> Result<(), Error> { + // Getting a multipart reader wrapper + let mut multipart = Multipart::from_request(request).unwrap(); + // Fetching all data and processing it + match multipart.save_all() { + SaveResult::Full(entries) => process_entries(entries), + SaveResult::Partial(entries, error) => { + try!(process_entries(entries)); + Err(error) + }, + SaveResult::Error(error) => Err(error) + } +} use std::fs::File; use std::io::{Error, Read}; fn process_entries(entries: Entries) -> Result<(), Error> { From 09ef5441e0f926c66641910134cb866f808fd25f Mon Sep 17 00:00:00 2001 From: White-Oak Date: Sat, 19 Mar 2016 21:17:41 +0300 Subject: [PATCH 133/453] Run cargo fmt --- multipart/samples/tiny_http/src/main.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/multipart/samples/tiny_http/src/main.rs b/multipart/samples/tiny_http/src/main.rs index 1489f3992..15e28a668 100644 --- a/multipart/samples/tiny_http/src/main.rs +++ b/multipart/samples/tiny_http/src/main.rs @@ -13,7 +13,11 @@ fn main() { // Answering with an HTTP OK and a string let response_string = "Multipart data is received!".as_bytes(); - let response = Response::new(StatusCode(200), vec![], response_string, Some(response_string.len()), None); + let response = Response::new(StatusCode(200), + vec![], + response_string, + Some(response_string.len()), + None); request.respond(response).unwrap(); } } @@ -27,8 +31,8 @@ fn process_multipart(request: &mut Request) -> Result<(), Error> { SaveResult::Partial(entries, error) => { try!(process_entries(entries)); Err(error) - }, - SaveResult::Error(error) => Err(error) + } + SaveResult::Error(error) => Err(error), } } use std::fs::File; @@ -41,7 +45,7 @@ fn process_entries(entries: Entries) -> Result<(), Error> { for (name, savedfile) in entries.files { let filename = match savedfile.filename { Some(s) => s, - None => "None".into() + None => "None".into(), }; let mut file = try!(File::open(savedfile.path)); let mut contents = String::new(); From 91312933982d9529d0e33f1a88c11c1d010cdf1e Mon Sep 17 00:00:00 2001 From: hatsunearu Date: Sat, 19 Mar 2016 23:01:07 -0400 Subject: [PATCH 134/453] Fix typo that breaks the link to the Hyper repo --- multipart/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/src/lib.rs b/multipart/src/lib.rs index 0d39bcd83..2ce159dbd 100644 --- a/multipart/src/lib.rs +++ b/multipart/src/lib.rs @@ -14,7 +14,7 @@ //! * `server` (default): Enable the server-side abstractions for multipart requests. If the //! `hyper` feature is also set, enables integration with the Hyper HTTP server API. //! -//! * `hyper` (default): Enable integration with the [Hyper](https:://github.com/hyperium/hyper) HTTP library +//! * `hyper` (default): Enable integration with the [Hyper](https://github.com/hyperium/hyper) HTTP library //! for client and/or server depending on which other feature flags are set. //! //! * `iron`: Enable integration with the [Iron](http://ironframework.io) web application From cfd31ac976ee1c42bbc27260bfe68fe51cf4cc3b Mon Sep 17 00:00:00 2001 From: White-Oak Date: Tue, 22 Mar 2016 15:10:00 +0300 Subject: [PATCH 135/453] Refactored tiny_http sample --- multipart/samples/tiny_http/src/main.rs | 71 ++++++++++++++++--------- 1 file changed, 47 insertions(+), 24 deletions(-) diff --git a/multipart/samples/tiny_http/src/main.rs b/multipart/samples/tiny_http/src/main.rs index 15e28a668..4827d75f6 100644 --- a/multipart/samples/tiny_http/src/main.rs +++ b/multipart/samples/tiny_http/src/main.rs @@ -1,43 +1,56 @@ extern crate tiny_http; extern crate multipart; +use std::fs::File; +use std::io::{Error, Read}; use multipart::server::{Multipart, Entries, SaveResult}; use tiny_http::{Response, StatusCode, Request}; fn main() { // Starting a server on `localhost:80` - let server = tiny_http::Server::http("localhost:80").unwrap(); + let server = tiny_http::Server::http("localhost:80").expect("Could not bind localhost:80"); loop { // This blocks until the next request is received let mut request = server.recv().unwrap(); - process_multipart(&mut request).unwrap(); - - // Answering with an HTTP OK and a string - let response_string = "Multipart data is received!".as_bytes(); - let response = Response::new(StatusCode(200), - vec![], - response_string, - Some(response_string.len()), - None); - request.respond(response).unwrap(); + + // Processes a request and returns response or an occured error + let result = process_request(&mut request); + let resp = match result { + Ok(resp) => resp, + Err(e) => { + println!("An error has occured during request proccessing: {:?}", e); + build_response(500, "The received data was not correctly proccessed on the server") + } + }; + + // Answers with a response to a client + request.respond(resp).unwrap(); } } -fn process_multipart(request: &mut Request) -> Result<(), Error> { +/// Processes a request and returns response or an occured error. +fn process_request<'a, 'b>(request: &'a mut Request) -> Result, Error> { // Getting a multipart reader wrapper - let mut multipart = Multipart::from_request(request).unwrap(); - // Fetching all data and processing it - match multipart.save_all() { - SaveResult::Full(entries) => process_entries(entries), - SaveResult::Partial(entries, error) => { - try!(process_entries(entries)); - Err(error) + match Multipart::from_request(request) { + Ok(mut multipart) => { + // Fetching all data and processing it. + // save_all() reads the request fully, parsing all fields and saving all files + // in a new temporary directory under the OS temporary directory. + match multipart.save_all() { + SaveResult::Full(entries) => process_entries(entries), + SaveResult::Partial(entries, error) => { + try!(process_entries(entries)); + Err(error) + } + SaveResult::Error(error) => Err(error), + } } - SaveResult::Error(error) => Err(error), + Err(_) => Ok(build_response(400, "The request is not multipart")), } } -use std::fs::File; -use std::io::{Error, Read}; -fn process_entries(entries: Entries) -> Result<(), Error> { + +/// Processes saved entries from multipart request. +/// Returns an OK response or an error. +fn process_entries<'a>(entries: Entries) -> Result, Error> { for (name, field) in entries.fields { println!(r#"Field "{}": "{}""#, name, field); } @@ -54,5 +67,15 @@ fn process_entries(entries: Entries) -> Result<(), Error> { println!(r#"Field "{}" is file "{}":"#, name, filename); println!("{}", contents); } - Ok(()) + Ok(build_response(200, "Multipart data is received!".into())) +} + +/// A utility function to build responses using only two arguments +fn build_response(status_code: u16, response: &str) -> Response<&[u8]> { + let bytes = response.as_bytes(); + Response::new(StatusCode(status_code), + vec![], + bytes, + Some(bytes.len()), + None) } From 9a3a6aa3630a45b99dc3e66b72a4253152ff898f Mon Sep 17 00:00:00 2001 From: White-Oak Date: Wed, 23 Mar 2016 04:03:05 +0300 Subject: [PATCH 136/453] Implemented an iron sample --- multipart/samples/iron/.gitignore | 1 + multipart/samples/iron/Cargo.lock | 494 +++++++++++++++++++++++++++++ multipart/samples/iron/Cargo.toml | 12 + multipart/samples/iron/src/main.rs | 60 ++++ 4 files changed, 567 insertions(+) create mode 100644 multipart/samples/iron/.gitignore create mode 100644 multipart/samples/iron/Cargo.lock create mode 100644 multipart/samples/iron/Cargo.toml create mode 100644 multipart/samples/iron/src/main.rs diff --git a/multipart/samples/iron/.gitignore b/multipart/samples/iron/.gitignore new file mode 100644 index 000000000..2f7896d1d --- /dev/null +++ b/multipart/samples/iron/.gitignore @@ -0,0 +1 @@ +target/ diff --git a/multipart/samples/iron/Cargo.lock b/multipart/samples/iron/Cargo.lock new file mode 100644 index 000000000..f0bc97f73 --- /dev/null +++ b/multipart/samples/iron/Cargo.lock @@ -0,0 +1,494 @@ +[root] +name = "iron" +version = "0.1.0" +dependencies = [ + "iron 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", + "multipart 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "aho-corasick" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "memchr 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "bitflags" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "buf_redux" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "conduit-mime-types" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rustc-serialize 0.3.18 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "cookie" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "openssl 0.7.8 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc-serialize 0.3.18 (registry+https://github.com/rust-lang/crates.io-index)", + "time 0.1.34 (registry+https://github.com/rust-lang/crates.io-index)", + "url 0.5.7 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "env_logger" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "log 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "regex 0.1.58 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "error" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "traitobject 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)", + "typeable 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "gcc" +version = "0.3.26" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "gdi32-sys" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "hpack" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "log 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "httparse" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "hyper" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cookie 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "httparse 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "language-tags 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "mime 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "num_cpus 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "openssl 0.7.8 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc-serialize 0.3.18 (registry+https://github.com/rust-lang/crates.io-index)", + "solicit 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)", + "time 0.1.34 (registry+https://github.com/rust-lang/crates.io-index)", + "traitobject 0.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "typeable 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "unicase 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "url 0.5.7 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "iron" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "conduit-mime-types 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)", + "error 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", + "hyper 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 0.1.15 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "modifier 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "num_cpus 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "plugin 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", + "typemap 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "url 0.5.7 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "kernel32-sys" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "language-tags" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "lazy_static" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "libc" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "libressl-pnacl-sys" +version = "2.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "pnacl-build-helper 1.4.10 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "log" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "matches" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "memchr" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "mime" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "log 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 0.6.15 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "mime_guess" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "mime 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "phf 0.7.13 (registry+https://github.com/rust-lang/crates.io-index)", + "phf_codegen 0.7.13 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "modifier" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "multipart" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "buf_redux 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "env_logger 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "iron 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", + "mime 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", + "mime_guess 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", + "tempdir 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "num" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "num_cpus" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "openssl" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bitflags 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "gcc 0.3.26 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 0.1.15 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "openssl-sys 0.7.8 (registry+https://github.com/rust-lang/crates.io-index)", + "openssl-sys-extras 0.7.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "openssl-sys" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "gdi32-sys 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "libressl-pnacl-sys 2.1.6 (registry+https://github.com/rust-lang/crates.io-index)", + "pkg-config 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", + "user32-sys 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "openssl-sys-extras" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "gcc 0.3.26 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "openssl-sys 0.7.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "phf" +version = "0.7.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "phf_shared 0.7.13 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "phf_codegen" +version = "0.7.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "phf_generator 0.7.13 (registry+https://github.com/rust-lang/crates.io-index)", + "phf_shared 0.7.13 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "phf_generator" +version = "0.7.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "phf_shared 0.7.13 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "phf_shared" +version = "0.7.13" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "pkg-config" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "plugin" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "typemap 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "pnacl-build-helper" +version = "1.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "tempdir 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rand" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "regex" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "aho-corasick 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", + "regex-syntax 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "utf8-ranges 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "regex-syntax" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "rustc-serialize" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "rustc_version" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "semver 0.1.20 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "semver" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "serde" +version = "0.6.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "num 0.1.31 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "solicit" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "hpack 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "tempdir" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "time" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "kernel32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "traitobject" +version = "0.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "traitobject" +version = "0.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "typeable" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "typemap" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unsafe-any 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "unicase" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rustc_version 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "unicode-bidi" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "matches 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "unicode-normalization" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "unsafe-any" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "traitobject 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "url" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "matches 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc-serialize 0.3.18 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-bidi 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-normalization 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "uuid 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "user32-sys" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "utf8-ranges" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "uuid" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", + "rustc-serialize 0.3.18 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "winapi" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "winapi-build" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + diff --git a/multipart/samples/iron/Cargo.toml b/multipart/samples/iron/Cargo.toml new file mode 100644 index 000000000..c1db1870a --- /dev/null +++ b/multipart/samples/iron/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "iron" +version = "0.1.0" +authors = ["White-Oak "] + +[dependencies] +iron = "0.2" + +[dependencies.multipart] +version = "0.5.1" +default-features = false +features = ["server", "iron"] diff --git a/multipart/samples/iron/src/main.rs b/multipart/samples/iron/src/main.rs new file mode 100644 index 000000000..f36d5e662 --- /dev/null +++ b/multipart/samples/iron/src/main.rs @@ -0,0 +1,60 @@ +extern crate multipart; +extern crate iron; + +use std::fs::File; +use std::io::{Read}; +use multipart::server::{Multipart, Entries, SaveResult}; +use iron::prelude::*; +use iron::status; + +fn main(){ + Iron::new(process_request).http("localhost:80").unwrap(); +} + +/// Processes a request and returns response or an occured error. +fn process_request(request: &mut Request) -> IronResult { + // Getting a multipart reader wrapper + match Multipart::from_request(request) { + Ok(mut multipart) => { + // Fetching all data and processing it. + // save_all() reads the request fully, parsing all fields and saving all files + // in a new temporary directory under the OS temporary directory. + match multipart.save_all() { + SaveResult::Full(entries) => process_entries(entries), + SaveResult::Partial(entries, error) => { + try!(process_entries(entries)); + Err(IronError::new(error, status::BadRequest)) + } + SaveResult::Error(error) => Err(IronError::new(error, status::BadRequest)), + } + } + Err(_) => Ok(Response::with((status::BadRequest, "The request is not multipart"))), + } +} + +/// Processes saved entries from multipart request. +/// Returns an OK response or an error. +fn process_entries(entries: Entries) -> IronResult { + for (name, field) in entries.fields { + println!(r#"Field "{}": "{}""#, name, field); + } + + for (name, savedfile) in entries.files { + let filename = match savedfile.filename { + Some(s) => s, + None => "None".into(), + }; + let mut file = match File::open(savedfile.path) { + Ok(file) => file, + Err(error) => return Err(IronError::new(error, status::BadRequest)) + }; + let mut contents = String::new(); + if let Err(error) = file.read_to_string(&mut contents) { + return Err(IronError::new(error, status::BadRequest)) + } + + println!(r#"Field "{}" is file "{}":"#, name, filename); + println!("{}", contents); + } + Ok(Response::with((status::Ok, "Multipart data is processed"))) +} From 6a5498f1a3e0cc66ea771a7c5605f73c1f4332a1 Mon Sep 17 00:00:00 2001 From: White-Oak Date: Wed, 23 Mar 2016 04:32:46 +0300 Subject: [PATCH 137/453] Added error messages --- multipart/samples/iron/src/main.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/multipart/samples/iron/src/main.rs b/multipart/samples/iron/src/main.rs index f36d5e662..0e5f4be3b 100644 --- a/multipart/samples/iron/src/main.rs +++ b/multipart/samples/iron/src/main.rs @@ -8,7 +8,7 @@ use iron::prelude::*; use iron::status; fn main(){ - Iron::new(process_request).http("localhost:80").unwrap(); + Iron::new(process_request).http("localhost:80").expect("Could not bind localhost:80"); } /// Processes a request and returns response or an occured error. @@ -23,12 +23,12 @@ fn process_request(request: &mut Request) -> IronResult { SaveResult::Full(entries) => process_entries(entries), SaveResult::Partial(entries, error) => { try!(process_entries(entries)); - Err(IronError::new(error, status::BadRequest)) + Err(IronError::new(error, status::InternalServerError)) } - SaveResult::Error(error) => Err(IronError::new(error, status::BadRequest)), + SaveResult::Error(error) => Err(IronError::new(error, status::InternalServerError)), } } - Err(_) => Ok(Response::with((status::BadRequest, "The request is not multipart"))), + Err(_) => Ok(Response::with((status::BadRequest, (status::BadRequest, "The request is not multipart")))), } } @@ -46,11 +46,11 @@ fn process_entries(entries: Entries) -> IronResult { }; let mut file = match File::open(savedfile.path) { Ok(file) => file, - Err(error) => return Err(IronError::new(error, status::BadRequest)) + Err(error) => return Err(IronError::new(error, (status::InternalServerError, "Server couldn't save file"))) }; let mut contents = String::new(); if let Err(error) = file.read_to_string(&mut contents) { - return Err(IronError::new(error, status::BadRequest)) + return Err(IronError::new(error, (status::BadRequest, "The file was not a text"))) } println!(r#"Field "{}" is file "{}":"#, name, filename); From c7c6a65db9df5c75c32fc17f84e7897489d6c81b Mon Sep 17 00:00:00 2001 From: White-Oak Date: Wed, 23 Mar 2016 04:33:34 +0300 Subject: [PATCH 138/453] Run cargo fmt --- multipart/samples/iron/src/main.rs | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/multipart/samples/iron/src/main.rs b/multipart/samples/iron/src/main.rs index 0e5f4be3b..5617aa30b 100644 --- a/multipart/samples/iron/src/main.rs +++ b/multipart/samples/iron/src/main.rs @@ -2,12 +2,12 @@ extern crate multipart; extern crate iron; use std::fs::File; -use std::io::{Read}; +use std::io::Read; use multipart::server::{Multipart, Entries, SaveResult}; use iron::prelude::*; use iron::status; -fn main(){ +fn main() { Iron::new(process_request).http("localhost:80").expect("Could not bind localhost:80"); } @@ -28,7 +28,10 @@ fn process_request(request: &mut Request) -> IronResult { SaveResult::Error(error) => Err(IronError::new(error, status::InternalServerError)), } } - Err(_) => Ok(Response::with((status::BadRequest, (status::BadRequest, "The request is not multipart")))), + Err(_) => { + Ok(Response::with((status::BadRequest, + (status::BadRequest, "The request is not multipart")))) + } } } @@ -46,11 +49,15 @@ fn process_entries(entries: Entries) -> IronResult { }; let mut file = match File::open(savedfile.path) { Ok(file) => file, - Err(error) => return Err(IronError::new(error, (status::InternalServerError, "Server couldn't save file"))) + Err(error) => { + return Err(IronError::new(error, + (status::InternalServerError, + "Server couldn't save file"))) + } }; let mut contents = String::new(); if let Err(error) = file.read_to_string(&mut contents) { - return Err(IronError::new(error, (status::BadRequest, "The file was not a text"))) + return Err(IronError::new(error, (status::BadRequest, "The file was not a text"))); } println!(r#"Field "{}" is file "{}":"#, name, filename); From 620ff2fdfb0948c9a9241b769bcf060bfa4a767a Mon Sep 17 00:00:00 2001 From: White-Oak Date: Wed, 23 Mar 2016 04:45:42 +0300 Subject: [PATCH 139/453] Fixed a typo --- multipart/samples/iron/src/main.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/multipart/samples/iron/src/main.rs b/multipart/samples/iron/src/main.rs index 5617aa30b..362b399cb 100644 --- a/multipart/samples/iron/src/main.rs +++ b/multipart/samples/iron/src/main.rs @@ -29,8 +29,7 @@ fn process_request(request: &mut Request) -> IronResult { } } Err(_) => { - Ok(Response::with((status::BadRequest, - (status::BadRequest, "The request is not multipart")))) + Ok(Response::with((status::BadRequest, "The request is not multipart"))) } } } From 0a46663e3f87163e1253eee3f543c1ced8036bc5 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Mon, 28 Mar 2016 15:39:57 -0700 Subject: [PATCH 140/453] Update copyright disclaimer in MIT license --- multipart/LICENSE-MIT | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/LICENSE-MIT b/multipart/LICENSE-MIT index 25597d583..3145ad975 100644 --- a/multipart/LICENSE-MIT +++ b/multipart/LICENSE-MIT @@ -1,4 +1,4 @@ -Copyright (c) 2010 The Rust Project Developers +Copyright (c) 2016 The `multipart` Crate Developers Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated From dcecac267c1f00372dfdf2a049f26f7db7575a7a Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Mon, 28 Mar 2016 15:54:33 -0700 Subject: [PATCH 141/453] Create README for samples --- multipart/samples/README.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 multipart/samples/README.md diff --git a/multipart/samples/README.md b/multipart/samples/README.md new file mode 100644 index 000000000..76f74ba96 --- /dev/null +++ b/multipart/samples/README.md @@ -0,0 +1,25 @@ +`multipart` Sample Projects +=========================== + +These are simple but fully fledged Cargo/Rust application projects to show how to use `multipart` with the various crates it integrates with. + +These projects carry the same licenses as [`multipart` itself](https://github.com/cybergeek94/multipart#license), though this may be lightened to a copyright-free license in the near future. + +More sample projects are underway and volunteers to create them are still needed. See [this issue](https://github.com/cybergeek94/multipart/issues/29) for more information. + +[`iron`](samples/iron) +----- +Author: [White-Oak][white-oak] + +This sample project shows how to use `multipart` with the [Iron web application framework](http://ironframework.io/), via `multipart`'s support +for the `iron::Request` type. + + +[`tiny_http`](samples/tiny_http) +---------- +Author: [White-Oak](white-oak) + +This sample project shows how to use `multipart` with the [`tiny_http` crate](https://crates.io/crates/tiny_http), via `multipart`'s support for the `tiny_http::Request` type. + + +[white-oak]: https://github.com/white-oak From cdac108469a19aa57805df900947962078c674e8 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Mon, 28 Mar 2016 15:55:47 -0700 Subject: [PATCH 142/453] Fix sample project links I *knew* the previewer was using the wrong base URL for relative links --- multipart/samples/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/multipart/samples/README.md b/multipart/samples/README.md index 76f74ba96..88c53b0be 100644 --- a/multipart/samples/README.md +++ b/multipart/samples/README.md @@ -7,7 +7,7 @@ These projects carry the same licenses as [`multipart` itself](https://github.co More sample projects are underway and volunteers to create them are still needed. See [this issue](https://github.com/cybergeek94/multipart/issues/29) for more information. -[`iron`](samples/iron) +[`iron`](iron) ----- Author: [White-Oak][white-oak] @@ -15,7 +15,7 @@ This sample project shows how to use `multipart` with the [Iron web application for the `iron::Request` type. -[`tiny_http`](samples/tiny_http) +[`tiny_http`](tiny_http) ---------- Author: [White-Oak](white-oak) From 46b2c2ee594c0973a103e864ce8c6a0c40c7cbf1 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Mon, 28 Mar 2016 16:00:43 -0700 Subject: [PATCH 143/453] Add link to samples --- multipart/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/multipart/README.md b/multipart/README.md index 992cedf7c..b6d79282f 100644 --- a/multipart/README.md +++ b/multipart/README.md @@ -9,6 +9,8 @@ Supports several different HTTP crates. ##Integrations +Sample projects demonstrating how to use `multipart` with these crates are available under [`samples/`](samples). + ####[Hyper](http://hyper.rs) via the `hyper` feature (enabled by default). From c3a4384fe2a75ce4bfb4862e287deba3504ccf4b Mon Sep 17 00:00:00 2001 From: Andre Bogus Date: Sun, 3 Apr 2016 00:40:15 +0200 Subject: [PATCH 144/453] fixed two clippy warnings --- multipart/src/client/lazy.rs | 2 +- multipart/src/server/mod.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/multipart/src/client/lazy.rs b/multipart/src/client/lazy.rs index ae88db38c..d677f9c59 100644 --- a/multipart/src/client/lazy.rs +++ b/multipart/src/client/lazy.rs @@ -251,7 +251,7 @@ pub struct PreparedFields<'d> { impl<'d> PreparedFields<'d> { fn from_fields<'n>(fields: &mut Vec>, boundary: String, buffer_threshold: Option) -> Result> { - let buffer_threshold = buffer_threshold.unwrap_or(u64::max_value()); + let buffer_threshold = buffer_threshold.unwrap_or(::std::u64::MAX); let mut prep_fields = Vec::with_capacity(fields.len()); diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index 311874ddd..d8251da01 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -202,13 +202,13 @@ impl Multipart { let _ = try!(self.source.read(&mut out)); if *b"\r\n" == out { - return Ok(true); + Ok(true) } else { if *b"--" != out { warn!("Unexpected 2-bytes after boundary: {:?}", out); } - return Ok(false); + Ok(false) } } } From 53548d28d553d95f423ecaa8a7d81d6918ad090d Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Tue, 22 Mar 2016 21:16:00 -0700 Subject: [PATCH 145/453] Add lorem ipsum sample text --- multipart/samples/lorem_ipsum.txt | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 multipart/samples/lorem_ipsum.txt diff --git a/multipart/samples/lorem_ipsum.txt b/multipart/samples/lorem_ipsum.txt new file mode 100644 index 000000000..1704658d6 --- /dev/null +++ b/multipart/samples/lorem_ipsum.txt @@ -0,0 +1,7 @@ +Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed dignissim, lectus a placerat vestibulum, mi massa dapibus ante, placerat efficitur dui est non lorem. Donec metus lacus, ultricies id suscipit sed, varius et augue. Vivamus erat lectus, interdum in quam non, euismod venenatis eros. Sed vitae magna orci. Duis finibus velit sem, eu luctus urna fringilla vitae. Suspendisse volutpat eros a tincidunt porttitor. Sed ut massa pretium, tempor neque nec, lobortis quam. Etiam vestibulum mauris eu sem consectetur, condimentum fermentum libero vulputate. Vestibulum porttitor leo et blandit condimentum. Pellentesque auctor odio eros, nec placerat lorem ultrices vitae. Suspendisse pretium tellus a ipsum sagittis consequat. Nullam pulvinar ligula ut fermentum laoreet. Maecenas rhoncus ut neque vitae tincidunt. Maecenas tincidunt at orci sed scelerisque. Sed porttitor tincidunt purus, ut efficitur leo lobortis vitae. Aenean et orci dolor. + +Vestibulum at laoreet felis. Cras et justo libero. Morbi pulvinar tincidunt odio, id finibus magna tincidunt non. Nulla facilisi. In at finibus lacus. Phasellus non volutpat dui. Vivamus porta fermentum dignissim. Nulla facilisi. Mauris laoreet semper ex lacinia interdum. Donec et dui non orci cursus scelerisque vulputate non neque. Fusce efficitur maximus turpis tempor interdum. Proin sit amet nunc pretium, varius dui sed, pretium nulla. Integer commodo orci ut felis bibendum feugiat. + +In interdum pulvinar tellus, quis porta eros consectetur in. Ut pharetra sem quam, id congue urna tempus eu. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia Curae; Sed egestas mattis ex quis malesuada. Quisque justo enim, posuere non arcu id, dictum pulvinar ante. Curabitur at rhoncus turpis. Donec in odio ut dolor elementum ultricies. Integer massa lacus, ullamcorper non nisi sed, scelerisque commodo lacus. Aliquam erat volutpat. Etiam eleifend libero tincidunt lobortis dignissim. Aliquam in odio sed libero sollicitudin pharetra. + +In quis consectetur ex, nec tempus mi. Donec commodo urna augue, non hendrerit mi lobortis et. Duis a augue laoreet, pulvinar purus luctus, rhoncus est. Quisque sodales sollicitudin augue ac bibendum. Sed a metus risus. Nulla non nulla nisl. Aenean erat velit, tempor id pellentesque eu, volutpat vitae dolor. Praesent commodo, dui in luctus aliquet, est tortor vehicula nibh, sed sollicitudin dui elit eu purus. Integer lacinia rutrum convallis. Nullam varius fringilla dui, elementum finibus magna tincidunt id. Praesent et cursus purus, vitae blandit purus. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia Curae; Aliquam vel cursus neque. \ No newline at end of file From bca26e6423f7470bac7a40106f848f0b94157a19 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Tue, 22 Mar 2016 02:53:18 -0700 Subject: [PATCH 146/453] Upgrade to Hyper 0.8 --- multipart/Cargo.toml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index aaabe47be..6112c1f18 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "multipart" -version = "0.5.1" +version = "0.6.0" authors = ["Austin Bonander "] @@ -18,8 +18,8 @@ license = "MIT OR Apache-2.0" [dependencies] env_logger = "0.3" log = "0.3" -mime = "0.1" -mime_guess = "=1.5" +mime = "0.2" +mime_guess = "1.6" rand = "0.3" tempdir = "0.3" @@ -29,7 +29,7 @@ version = "0.1" [dependencies.hyper] optional = true -version = "0.7" +version = "0.8" [dependencies.iron] optional = true From 54f54370c9e0ea9888b276be9f550e4da7da51bb Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Tue, 22 Mar 2016 02:55:28 -0700 Subject: [PATCH 147/453] Create alpha version --- multipart/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 6112c1f18..641c598cd 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "multipart" -version = "0.6.0" +version = "0.6.0-alpha" authors = ["Austin Bonander "] From 35bedaa959232110630ec9502a290e5e5bac1e07 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Thu, 7 Apr 2016 17:55:32 -0700 Subject: [PATCH 148/453] Bump Iron version --- multipart/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 641c598cd..ab5e25ad6 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -33,7 +33,7 @@ version = "0.8" [dependencies.iron] optional = true -version = "0.2" +version = "0.3" [dependencies.memchr] optional = true From 40212ef0c41ae0225be9920bb9286f9e38f0f06a Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Thu, 7 Apr 2016 17:56:03 -0700 Subject: [PATCH 149/453] Re-enable Nickel @ 0.8 --- multipart/Cargo.toml | 7 ++++--- multipart/src/server/mod.rs | 2 +- multipart/src/server/nickel.rs | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index ab5e25ad6..986f1fc38 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -39,9 +39,10 @@ version = "0.3" optional = true version = "0.1" +# NOTE: use `nickel_` feature, as `hyper` feature is required. [dependencies.nickel] optional = true -version = "0.7" +version = "0.8" [dependencies.tiny_http] optional = true @@ -51,5 +52,5 @@ version = "0.5" client = [] default = ["hyper", "server", "client"] server = ["buf_redux", "memchr"] -#Disabled until Nickel upgrades to Hyper 0.7 -#nickel2 = ["hyper", "nickel"] +nickel_ = ["nickel", "hyper"] +all = ["iron", "nickel_", "tiny_http"] diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index d8251da01..4cd3ad330 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -41,7 +41,7 @@ pub mod hyper; #[cfg(feature = "iron")] pub mod iron; -#[cfg(feature = "nickel2")] +#[cfg(feature = "nickel")] mod nickel; #[cfg(feature = "tiny_http")] diff --git a/multipart/src/server/nickel.rs b/multipart/src/server/nickel.rs index 1bcf3a38c..3f6d546b1 100644 --- a/multipart/src/server/nickel.rs +++ b/multipart/src/server/nickel.rs @@ -6,7 +6,7 @@ use hyper::server::Request as HyperRequest; use super::HttpRequest; -impl<'r, 'mw, 'server, D: 'mw> HttpRequest for &'r Request<'mw, 'server, D> { +impl<'r, 'mw, 'server, D: 'mw> HttpRequest for &'r mut Request<'mw, 'server, D> { type Body = &'r mut HyperRequest<'mw, 'server>; fn multipart_boundary(&self) -> Option<&str> { From f351dfab3ff2cb2be2f6891d32ce3ab0677e9c60 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Thu, 7 Apr 2016 17:59:40 -0700 Subject: [PATCH 150/453] Update README with Nickel integration --- multipart/README.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/multipart/README.md b/multipart/README.md index b6d79282f..15729165a 100644 --- a/multipart/README.md +++ b/multipart/README.md @@ -20,16 +20,21 @@ as integration with the new `hyper::Client` API via `multipart::client::lazy::Mu Server integration for `hyper::server::Request` via `multipart::server::Multipart`. ####[Iron](http://ironframework.io) -via the `iron` feature (new in 0.5). +via the `iron` feature. Provides regular server-side integration with `iron::Request` via `multipart::server::Multipart`, as well as a convenient `BeforeMiddleware` implementation in `multipart::server::iron::Intercept`. ####[tiny\_http](https://crates.io/crates/tiny_http/) -via the `tiny_http` feature (new in 0.5). +via the `tiny_http` feature. Provides server-side integration with `tiny_http::Request` via `multipart::server::Multipart`. +####[Nickel](http://http://nickel.rs/) ^((New!)) +via the `nickel_` feature + +Provides server-side integration with `&mut nickel::Request` via `multipart::server::Multipart`. + License ------- From c0d49385362fae1f479809c0ef74c43350aad4f9 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Thu, 7 Apr 2016 18:02:11 -0700 Subject: [PATCH 151/453] Update .travis.yml --- multipart/.travis.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/multipart/.travis.yml b/multipart/.travis.yml index 4d3a1c5f9..3bbfc1dc7 100644 --- a/multipart/.travis.yml +++ b/multipart/.travis.yml @@ -6,13 +6,10 @@ rust: os: - linux - osx -env: - global: - - FEATURES="iron tiny_http" script: - - cargo build -v --features "$FEATURES" - - cargo test -v --features "$FEATURES" - - cargo doc -v --no-deps --features "$FEATURES" + - cargo build -v --features all + - cargo test -v --features all + - cargo doc -v --no-deps --features all after_success: - | test ${TRAVIS_PULL_REQUEST} == "false" && \ From 6dd517f3ba6a2cb064a5372e27989386417ee02e Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Thu, 7 Apr 2016 18:05:51 -0700 Subject: [PATCH 152/453] Fix (New!) --- multipart/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/README.md b/multipart/README.md index 15729165a..531ea7068 100644 --- a/multipart/README.md +++ b/multipart/README.md @@ -30,7 +30,7 @@ via the `tiny_http` feature. Provides server-side integration with `tiny_http::Request` via `multipart::server::Multipart`. -####[Nickel](http://http://nickel.rs/) ^((New!)) +####[Nickel](http://http://nickel.rs/) New! via the `nickel_` feature Provides server-side integration with `&mut nickel::Request` via `multipart::server::Multipart`. From 619218786eb374ada799ff7c52c036fe686a4573 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Thu, 7 Apr 2016 18:06:20 -0700 Subject: [PATCH 153/453] Fix (New!) --- multipart/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/README.md b/multipart/README.md index 531ea7068..35d854a0b 100644 --- a/multipart/README.md +++ b/multipart/README.md @@ -30,7 +30,7 @@ via the `tiny_http` feature. Provides server-side integration with `tiny_http::Request` via `multipart::server::Multipart`. -####[Nickel](http://http://nickel.rs/) New! +####[Nickel](http://http://nickel.rs/) (New in 0.6!) via the `nickel_` feature Provides server-side integration with `&mut nickel::Request` via `multipart::server::Multipart`. From d2b368977ba27cbd768649b6182aea267abdad08 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Thu, 7 Apr 2016 18:08:29 -0700 Subject: [PATCH 154/453] Fix Nickel link --- multipart/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/README.md b/multipart/README.md index 35d854a0b..022d5d2af 100644 --- a/multipart/README.md +++ b/multipart/README.md @@ -30,7 +30,7 @@ via the `tiny_http` feature. Provides server-side integration with `tiny_http::Request` via `multipart::server::Multipart`. -####[Nickel](http://http://nickel.rs/) (New in 0.6!) +####[Nickel](http://nickel.rs/) (New in 0.6!) via the `nickel_` feature Provides server-side integration with `&mut nickel::Request` via `multipart::server::Multipart`. From b14cf62e842d548cb7ca8e40fc4f2284457e759c Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Thu, 7 Apr 2016 18:21:39 -0700 Subject: [PATCH 155/453] 0.6.0 release --- multipart/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 986f1fc38..9e379dc3c 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "multipart" -version = "0.6.0-alpha" +version = "0.6.0" authors = ["Austin Bonander "] From 04968b47ea0cb5dd32a4353628ad15c74fb3d7a2 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Thu, 7 Apr 2016 19:09:48 -0700 Subject: [PATCH 156/453] 0.6.1: Documentation fixes --- multipart/Cargo.toml | 2 +- multipart/src/lib.rs | 4 ++++ multipart/src/server/mod.rs | 2 +- multipart/src/server/nickel.rs | 9 ++++++--- 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 9e379dc3c..57ba96345 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "multipart" -version = "0.6.0" +version = "0.6.1" authors = ["Austin Bonander "] diff --git a/multipart/src/lib.rs b/multipart/src/lib.rs index 2ce159dbd..1815bcd0c 100644 --- a/multipart/src/lib.rs +++ b/multipart/src/lib.rs @@ -22,6 +22,10 @@ //! //! * `tiny_http`: Enable integration with the [`tiny_http`](https://github.com/frewsxcv/tiny-http) //! crate. See the [`server::tiny_http`](server/tiny_http/index.html) module for more information. +//! +//! * `nickel_`: Enable integration with the [Nickel](http://nickel.rs) web application framework. +//! See the [`server::nickel`](server/nickel/index.html) module for more information. Enables the `hyper` +//! feature. #![warn(missing_docs)] #[macro_use] extern crate log; extern crate env_logger; diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index 4cd3ad330..d3023ecf6 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -42,7 +42,7 @@ pub mod hyper; pub mod iron; #[cfg(feature = "nickel")] -mod nickel; +pub mod nickel; #[cfg(feature = "tiny_http")] pub mod tiny_http; diff --git a/multipart/src/server/nickel.rs b/multipart/src/server/nickel.rs index 3f6d546b1..5ac4da1d6 100644 --- a/multipart/src/server/nickel.rs +++ b/multipart/src/server/nickel.rs @@ -1,12 +1,15 @@ -//! Trait impl for Nickel, piggybacking on Hyper's integration. +//! Server-side integration with [Nickel](http://nickel.rs/) via the `nickel_` feature +//! (optional, enables `hyper` feature). +//! +//! Not shown here: [`impl HttpRequest for &mut nickel::Request`](../trait.HttpRequest.html#implementors). -use nickel::Request; +use nickel::Request as NickelRequest; use hyper::server::Request as HyperRequest; use super::HttpRequest; -impl<'r, 'mw, 'server, D: 'mw> HttpRequest for &'r mut Request<'mw, 'server, D> { +impl<'r, 'mw, 'server, D: 'mw> HttpRequest for &'r mut NickelRequest<'mw, 'server, D> { type Body = &'r mut HyperRequest<'mw, 'server>; fn multipart_boundary(&self) -> Option<&str> { From 8927787e08e79ae8a9042f0b673f87b4d09129f0 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Thu, 7 Apr 2016 22:35:16 -0700 Subject: [PATCH 157/453] Add _limited versions of save_all*() --- multipart/src/server/mod.rs | 53 +++++++++++++++++++++++++++++++++---- 1 file changed, 48 insertions(+), 5 deletions(-) diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index d3023ecf6..ad75c027e 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -138,7 +138,7 @@ impl Multipart { Err(err) => return SaveResult::Error(err), }; - match self.read_to_entries(&mut entries) { + match self.read_to_entries(&mut entries, None) { Ok(()) => SaveResult::Full(entries), Err(err) => SaveResult::Partial(entries, err), } @@ -155,17 +155,60 @@ impl Multipart { Err(err) => return SaveResult::Error(err), }; - match self.read_to_entries(&mut entries) { + match self.read_to_entries(&mut entries, None) { Ok(()) => SaveResult::Full(entries), Err(err) => SaveResult::Partial(entries, err), } } - fn read_to_entries(&mut self, entries: &mut Entries) -> io::Result<()> { + /// Read the request fully, parsing all fields and saving all fields in a new temporary + /// directory under the OS temporary directory. + /// + /// Files larger than `limit` will be truncated to `limit`. + /// + /// If there is an error in reading the request, returns the partial result along with the + /// error. See [`SaveResult`](enum.saveresult.html) for more information. + pub fn save_all_limited(&mut self, limit: u64) -> SaveResult { + let mut entries = match Entries::new_tempdir() { + Ok(entries) => entries, + Err(err) => return SaveResult::Error(err), + }; + + match self.read_to_entries(&mut entries, Some(limit)) { + Ok(()) => SaveResult::Full(entries), + Err(err) => SaveResult::Partial(entries, err), + } + } + + /// Read the request fully, parsing all fields and saving all files in a new temporary + /// directory under `dir`. + /// + /// Files larger than `limit` will be truncated to `limit`. + /// + /// If there is an error in reading the request, returns the partial result along with the + /// error. See [`SaveResult`](enum.saveresult.html) for more information. + pub fn save_all_under_limited>(&mut self, dir: P, limit: u64) -> SaveResult { + let mut entries = match Entries::new_tempdir_in(dir) { + Ok(entries) => entries, + Err(err) => return SaveResult::Error(err), + }; + + match self.read_to_entries(&mut entries, Some(limit)) { + Ok(()) => SaveResult::Full(entries), + Err(err) => SaveResult::Partial(entries, err), + } + } + + fn read_to_entries(&mut self, entries: &mut Entries, limit: Option) -> io::Result<()> { while let Some(field) = try!(self.read_entry()) { match field.data { MultipartData::File(mut file) => { - let file = try!(file.save_in(&entries.dir)); + let file = if let Some(limit) = limit { + try!(file.save_in_limited(&entries.dir, limit)) + } else { + try!(file.save_in(&entries.dir)) + }; + entries.files.insert(field.name, file); }, MultipartData::Text(text) => { @@ -175,7 +218,7 @@ impl Multipart { } Ok(()) - } + } fn read_line(&mut self) -> io::Result<&str> { self.line_buf.clear(); From bd26743c4fdb34b91883eb6f05407b965121074b Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Thu, 7 Apr 2016 22:51:38 -0700 Subject: [PATCH 158/453] 0.7.0: Add _limited variants to .save_all*() --- multipart/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 57ba96345..f14a27e9b 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "multipart" -version = "0.6.1" +version = "0.7.0" authors = ["Austin Bonander "] From 19aa88964c04db0df426185708bddb8399cc6bfa Mon Sep 17 00:00:00 2001 From: Robin Stocker Date: Thu, 9 Jun 2016 22:49:59 +1000 Subject: [PATCH 159/453] README: Fix typo "Mulitpart" --- multipart/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/README.md b/multipart/README.md index 022d5d2af..943e010b9 100644 --- a/multipart/README.md +++ b/multipart/README.md @@ -15,7 +15,7 @@ Sample projects demonstrating how to use `multipart` with these crates are avail via the `hyper` feature (enabled by default). Client integration includes support for regular `hyper::client::Request` objects via `multipart::client::Multipart`, as well -as integration with the new `hyper::Client` API via `multipart::client::lazy::Mulitpart` (new in 0.5). +as integration with the new `hyper::Client` API via `multipart::client::lazy::Multipart` (new in 0.5). Server integration for `hyper::server::Request` via `multipart::server::Multipart`. From c8449fa46bed776bad84dc069dc14c7503bc3725 Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 12 Jul 2016 21:57:16 -0400 Subject: [PATCH 160/453] start sample hyper server --- multipart/samples/README.md | 7 +++++ multipart/samples/hyper_server/.gitignore | 15 +++++++++++ multipart/samples/hyper_server/Cargo.toml | 8 ++++++ multipart/samples/hyper_server/src/main.rs | 31 ++++++++++++++++++++++ 4 files changed, 61 insertions(+) create mode 100644 multipart/samples/hyper_server/.gitignore create mode 100644 multipart/samples/hyper_server/Cargo.toml create mode 100644 multipart/samples/hyper_server/src/main.rs diff --git a/multipart/samples/README.md b/multipart/samples/README.md index 88c53b0be..7d1bf9965 100644 --- a/multipart/samples/README.md +++ b/multipart/samples/README.md @@ -21,5 +21,12 @@ Author: [White-Oak](white-oak) This sample project shows how to use `multipart` with the [`tiny_http` crate](https://crates.io/crates/tiny_http), via `multipart`'s support for the `tiny_http::Request` type. +[`hyper_server`](hyper_server) +----------------------------- +Author: [Puhrez](puhrez) +This sample project shows how to use `multipart` with a [`hyper::Server`] (http://hyper.rs/) to intercept multipart requests. + + +[puhrez]: https://github.com/puhrez [white-oak]: https://github.com/white-oak diff --git a/multipart/samples/hyper_server/.gitignore b/multipart/samples/hyper_server/.gitignore new file mode 100644 index 000000000..dc6181361 --- /dev/null +++ b/multipart/samples/hyper_server/.gitignore @@ -0,0 +1,15 @@ +/target +/Cargo.lock +*.swp +*~ +# Compiled files +*.o +*.so +*.rlib +*.dll + +# Executables +*.exe + +# Generated by Cargo +/target/ \ No newline at end of file diff --git a/multipart/samples/hyper_server/Cargo.toml b/multipart/samples/hyper_server/Cargo.toml new file mode 100644 index 000000000..05d06905e --- /dev/null +++ b/multipart/samples/hyper_server/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "hyper_server" +version = "0.1.0" +authors = ["Michael "] + +[dependencies] +hyper = "0.8" +multipart = "0.5.1" diff --git a/multipart/samples/hyper_server/src/main.rs b/multipart/samples/hyper_server/src/main.rs new file mode 100644 index 000000000..2c7115b17 --- /dev/null +++ b/multipart/samples/hyper_server/src/main.rs @@ -0,0 +1,31 @@ +extern crate hyper; +extern crate multipart; + +use hyper::server::{Handler, Server, Request, Response}; +use hyper::status::StatusCode::ImATeapot; +use multipart::server::hyper::{Switch, MultipartHandler}; +use multipart::server::{Multipart, Entries}; + +struct NonMultipart; +impl Handler for NonMultipart { + fn handle(&self, _: Request, mut res: Response) { + *res.status_mut() = ImATeapot; + res.send(b"Please send a multipart req :(\n").unwrap(); + } +} + +struct EchoMultipart; +impl MultipartHandler for EchoMultipart { + fn handle_multipart(&self, multipart: Multipart, res: Response) { + res.send(b"Thanks for the multipart req :)\n").unwrap(); + } +} + +fn main() { + Server::http("0.0.0.0:3333").unwrap().handle( + Switch::new( + NonMultipart, + EchoMultipart + )).unwrap(); + println!("Listening on 0.0.0.0:3333"); +} From a2bae58beae888d4233f4c30a5dbb27242cebd37 Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 12 Jul 2016 22:50:57 -0400 Subject: [PATCH 161/453] use HyperRequest for handle_multipart --- multipart/samples/hyper_server/src/main.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/multipart/samples/hyper_server/src/main.rs b/multipart/samples/hyper_server/src/main.rs index 2c7115b17..eeb11867c 100644 --- a/multipart/samples/hyper_server/src/main.rs +++ b/multipart/samples/hyper_server/src/main.rs @@ -3,7 +3,8 @@ extern crate multipart; use hyper::server::{Handler, Server, Request, Response}; use hyper::status::StatusCode::ImATeapot; -use multipart::server::hyper::{Switch, MultipartHandler}; +use hyper::server::response::Response as HyperResponse; +use multipart::server::hyper::{Switch, MultipartHandler, HyperRequest}; use multipart::server::{Multipart, Entries}; struct NonMultipart; @@ -16,7 +17,7 @@ impl Handler for NonMultipart { struct EchoMultipart; impl MultipartHandler for EchoMultipart { - fn handle_multipart(&self, multipart: Multipart, res: Response) { + fn handle_multipart(&self, multipart: Multipart, res: HyperResponse) { res.send(b"Thanks for the multipart req :)\n").unwrap(); } } From 731b302793a052f6c68c85ad26681992de31d5c2 Mon Sep 17 00:00:00 2001 From: Michael Date: Mon, 25 Jul 2016 13:44:44 -0400 Subject: [PATCH 162/453] complete hyper server example --- multipart/{samples => examples}/README.md | 15 + multipart/examples/hyper_server.rs | 69 +++ .../iron/src/main.rs => examples/iron.rs} | 0 .../{samples => examples}/lorem_ipsum.txt | 0 .../src/main.rs => examples/tiny_http.rs} | 0 multipart/samples/hyper_server/.gitignore | 15 - multipart/samples/hyper_server/Cargo.toml | 8 - multipart/samples/hyper_server/src/main.rs | 32 -- multipart/samples/iron/.gitignore | 1 - multipart/samples/iron/Cargo.lock | 494 ------------------ multipart/samples/iron/Cargo.toml | 12 - multipart/samples/tiny_http/.gitignore | 1 - multipart/samples/tiny_http/Cargo.lock | 314 ----------- multipart/samples/tiny_http/Cargo.toml | 12 - 14 files changed, 84 insertions(+), 889 deletions(-) rename multipart/{samples => examples}/README.md (88%) create mode 100644 multipart/examples/hyper_server.rs rename multipart/{samples/iron/src/main.rs => examples/iron.rs} (100%) rename multipart/{samples => examples}/lorem_ipsum.txt (100%) rename multipart/{samples/tiny_http/src/main.rs => examples/tiny_http.rs} (100%) delete mode 100644 multipart/samples/hyper_server/.gitignore delete mode 100644 multipart/samples/hyper_server/Cargo.toml delete mode 100644 multipart/samples/hyper_server/src/main.rs delete mode 100644 multipart/samples/iron/.gitignore delete mode 100644 multipart/samples/iron/Cargo.lock delete mode 100644 multipart/samples/iron/Cargo.toml delete mode 100644 multipart/samples/tiny_http/.gitignore delete mode 100644 multipart/samples/tiny_http/Cargo.lock delete mode 100644 multipart/samples/tiny_http/Cargo.toml diff --git a/multipart/samples/README.md b/multipart/examples/README.md similarity index 88% rename from multipart/samples/README.md rename to multipart/examples/README.md index 7d1bf9965..cc346a06d 100644 --- a/multipart/samples/README.md +++ b/multipart/examples/README.md @@ -14,6 +14,12 @@ Author: [White-Oak][white-oak] This sample project shows how to use `multipart` with the [Iron web application framework](http://ironframework.io/), via `multipart`'s support for the `iron::Request` type. +To run: + +``` +$ cargo run --features "iron" --example iron +``` + [`tiny_http`](tiny_http) ---------- @@ -21,12 +27,21 @@ Author: [White-Oak](white-oak) This sample project shows how to use `multipart` with the [`tiny_http` crate](https://crates.io/crates/tiny_http), via `multipart`'s support for the `tiny_http::Request` type. + +``` +$ cargo run --features "tiny_http" --example tiny_http +``` + [`hyper_server`](hyper_server) ----------------------------- Author: [Puhrez](puhrez) This sample project shows how to use `multipart` with a [`hyper::Server`] (http://hyper.rs/) to intercept multipart requests. +``` +$ cargo run --example hyper_server +``` + [puhrez]: https://github.com/puhrez [white-oak]: https://github.com/white-oak diff --git a/multipart/examples/hyper_server.rs b/multipart/examples/hyper_server.rs new file mode 100644 index 000000000..36e335157 --- /dev/null +++ b/multipart/examples/hyper_server.rs @@ -0,0 +1,69 @@ +extern crate hyper; +extern crate multipart; + +use std::fs::File; +use std::io::{Read, Result}; +use hyper::server::{Handler, Server, Request, Response}; +use hyper::status::StatusCode; +use hyper::server::response::Response as HyperResponse; +use multipart::server::hyper::{Switch, MultipartHandler, HyperRequest}; +use multipart::server::{Multipart, Entries, SaveResult}; + +struct NonMultipart; +impl Handler for NonMultipart { + fn handle(&self, _: Request, mut res: Response) { + *res.status_mut() = StatusCode::ImATeapot; + res.send(b"Please send a multipart req :(\n").unwrap(); + } +} + +struct EchoMultipart; +impl MultipartHandler for EchoMultipart { + fn handle_multipart(&self, mut multipart: Multipart, mut res: HyperResponse) { + let processing = match multipart.save_all() { + SaveResult::Full(entries) => process_entries(entries), + SaveResult::Partial(entries, error) => { + println!("Errors saving multipart:\n{:?}", error); + process_entries(entries) + } + SaveResult::Error(error) => { + println!("Errors saving multipart:\n{:?}", error); + Err(error) + } + }; + match processing { + Ok(_) => res.send(b"All good in the hood :)\n").unwrap(), + Err(_) => { + *res.status_mut() = StatusCode::BadRequest; + res.send(b"An error occurred :(\n").unwrap(); + } + } + } +} + +fn process_entries(entries: Entries) -> Result<()> { + for (name, field) in entries.fields { + print!(r#"Field "{}": "{}""#, name, field); + } + for (name, savedfile) in entries.files { + let filename = match savedfile.filename { + Some(s) => s, + None => "None".into() + }; + let mut file = try!(File::open(savedfile.path)); + let mut contents = String::new(); + try!(file.read_to_string(&mut contents)); + + println!(r#"Field "{}" is file "{}":"#, name, filename); + println!("{}", contents); + } + Ok(()) +} +fn main() { + println!("Listening on 0.0.0.0:3333"); + Server::http("0.0.0.0:3333").unwrap().handle( + Switch::new( + NonMultipart, + EchoMultipart + )).unwrap(); +} diff --git a/multipart/samples/iron/src/main.rs b/multipart/examples/iron.rs similarity index 100% rename from multipart/samples/iron/src/main.rs rename to multipart/examples/iron.rs diff --git a/multipart/samples/lorem_ipsum.txt b/multipart/examples/lorem_ipsum.txt similarity index 100% rename from multipart/samples/lorem_ipsum.txt rename to multipart/examples/lorem_ipsum.txt diff --git a/multipart/samples/tiny_http/src/main.rs b/multipart/examples/tiny_http.rs similarity index 100% rename from multipart/samples/tiny_http/src/main.rs rename to multipart/examples/tiny_http.rs diff --git a/multipart/samples/hyper_server/.gitignore b/multipart/samples/hyper_server/.gitignore deleted file mode 100644 index dc6181361..000000000 --- a/multipart/samples/hyper_server/.gitignore +++ /dev/null @@ -1,15 +0,0 @@ -/target -/Cargo.lock -*.swp -*~ -# Compiled files -*.o -*.so -*.rlib -*.dll - -# Executables -*.exe - -# Generated by Cargo -/target/ \ No newline at end of file diff --git a/multipart/samples/hyper_server/Cargo.toml b/multipart/samples/hyper_server/Cargo.toml deleted file mode 100644 index 05d06905e..000000000 --- a/multipart/samples/hyper_server/Cargo.toml +++ /dev/null @@ -1,8 +0,0 @@ -[package] -name = "hyper_server" -version = "0.1.0" -authors = ["Michael "] - -[dependencies] -hyper = "0.8" -multipart = "0.5.1" diff --git a/multipart/samples/hyper_server/src/main.rs b/multipart/samples/hyper_server/src/main.rs deleted file mode 100644 index eeb11867c..000000000 --- a/multipart/samples/hyper_server/src/main.rs +++ /dev/null @@ -1,32 +0,0 @@ -extern crate hyper; -extern crate multipart; - -use hyper::server::{Handler, Server, Request, Response}; -use hyper::status::StatusCode::ImATeapot; -use hyper::server::response::Response as HyperResponse; -use multipart::server::hyper::{Switch, MultipartHandler, HyperRequest}; -use multipart::server::{Multipart, Entries}; - -struct NonMultipart; -impl Handler for NonMultipart { - fn handle(&self, _: Request, mut res: Response) { - *res.status_mut() = ImATeapot; - res.send(b"Please send a multipart req :(\n").unwrap(); - } -} - -struct EchoMultipart; -impl MultipartHandler for EchoMultipart { - fn handle_multipart(&self, multipart: Multipart, res: HyperResponse) { - res.send(b"Thanks for the multipart req :)\n").unwrap(); - } -} - -fn main() { - Server::http("0.0.0.0:3333").unwrap().handle( - Switch::new( - NonMultipart, - EchoMultipart - )).unwrap(); - println!("Listening on 0.0.0.0:3333"); -} diff --git a/multipart/samples/iron/.gitignore b/multipart/samples/iron/.gitignore deleted file mode 100644 index 2f7896d1d..000000000 --- a/multipart/samples/iron/.gitignore +++ /dev/null @@ -1 +0,0 @@ -target/ diff --git a/multipart/samples/iron/Cargo.lock b/multipart/samples/iron/Cargo.lock deleted file mode 100644 index f0bc97f73..000000000 --- a/multipart/samples/iron/Cargo.lock +++ /dev/null @@ -1,494 +0,0 @@ -[root] -name = "iron" -version = "0.1.0" -dependencies = [ - "iron 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", - "multipart 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "aho-corasick" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "memchr 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "bitflags" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "buf_redux" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "conduit-mime-types" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "rustc-serialize 0.3.18 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "cookie" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "openssl 0.7.8 (registry+https://github.com/rust-lang/crates.io-index)", - "rustc-serialize 0.3.18 (registry+https://github.com/rust-lang/crates.io-index)", - "time 0.1.34 (registry+https://github.com/rust-lang/crates.io-index)", - "url 0.5.7 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "env_logger" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "log 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 0.1.58 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "error" -version = "0.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "traitobject 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)", - "typeable 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "gcc" -version = "0.3.26" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "gdi32-sys" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "winapi 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "hpack" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "log 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "httparse" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "hyper" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "cookie 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "httparse 1.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "language-tags 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", - "mime 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", - "num_cpus 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", - "openssl 0.7.8 (registry+https://github.com/rust-lang/crates.io-index)", - "rustc-serialize 0.3.18 (registry+https://github.com/rust-lang/crates.io-index)", - "solicit 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)", - "time 0.1.34 (registry+https://github.com/rust-lang/crates.io-index)", - "traitobject 0.0.1 (registry+https://github.com/rust-lang/crates.io-index)", - "typeable 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "unicase 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "url 0.5.7 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "iron" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "conduit-mime-types 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)", - "error 0.1.9 (registry+https://github.com/rust-lang/crates.io-index)", - "hyper 0.7.2 (registry+https://github.com/rust-lang/crates.io-index)", - "lazy_static 0.1.15 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", - "modifier 0.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "num_cpus 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", - "plugin 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", - "typemap 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", - "url 0.5.7 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "kernel32-sys" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "winapi 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "language-tags" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "lazy_static" -version = "0.1.15" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "libc" -version = "0.2.8" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "libressl-pnacl-sys" -version = "2.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "pnacl-build-helper 1.4.10 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "log" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "libc 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "matches" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "memchr" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "libc 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "mime" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "log 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 0.6.15 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "mime_guess" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "mime 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", - "phf 0.7.13 (registry+https://github.com/rust-lang/crates.io-index)", - "phf_codegen 0.7.13 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "modifier" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "multipart" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "buf_redux 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", - "env_logger 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", - "iron 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", - "memchr 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", - "mime 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", - "mime_guess 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)", - "rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", - "tempdir 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "num" -version = "0.1.31" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "num_cpus" -version = "0.2.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "libc 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "openssl" -version = "0.7.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "bitflags 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", - "gcc 0.3.26 (registry+https://github.com/rust-lang/crates.io-index)", - "lazy_static 0.1.15 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", - "openssl-sys 0.7.8 (registry+https://github.com/rust-lang/crates.io-index)", - "openssl-sys-extras 0.7.8 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "openssl-sys" -version = "0.7.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "gdi32-sys 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", - "libressl-pnacl-sys 2.1.6 (registry+https://github.com/rust-lang/crates.io-index)", - "pkg-config 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", - "user32-sys 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "openssl-sys-extras" -version = "0.7.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "gcc 0.3.26 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", - "openssl-sys 0.7.8 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "phf" -version = "0.7.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "phf_shared 0.7.13 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "phf_codegen" -version = "0.7.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "phf_generator 0.7.13 (registry+https://github.com/rust-lang/crates.io-index)", - "phf_shared 0.7.13 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "phf_generator" -version = "0.7.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "phf_shared 0.7.13 (registry+https://github.com/rust-lang/crates.io-index)", - "rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "phf_shared" -version = "0.7.13" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "pkg-config" -version = "0.3.8" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "plugin" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "typemap 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "pnacl-build-helper" -version = "1.4.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "tempdir 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "rand" -version = "0.3.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "libc 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "regex" -version = "0.1.58" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "aho-corasick 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", - "memchr 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", - "regex-syntax 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", - "utf8-ranges 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "regex-syntax" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "rustc-serialize" -version = "0.3.18" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "rustc_version" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "semver 0.1.20 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "semver" -version = "0.1.20" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "serde" -version = "0.6.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "num 0.1.31 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "solicit" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "hpack 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "tempdir" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "time" -version = "0.1.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "kernel32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "traitobject" -version = "0.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "traitobject" -version = "0.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "typeable" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "typemap" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "unsafe-any 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "unicase" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "rustc_version 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "unicode-bidi" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "matches 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "unicode-normalization" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "unsafe-any" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "traitobject 0.0.3 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "url" -version = "0.5.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "matches 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "rustc-serialize 0.3.18 (registry+https://github.com/rust-lang/crates.io-index)", - "unicode-bidi 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", - "unicode-normalization 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "uuid 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "user32-sys" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "winapi 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "utf8-ranges" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "uuid" -version = "0.1.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", - "rustc-serialize 0.3.18 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "winapi" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "winapi-build" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" - diff --git a/multipart/samples/iron/Cargo.toml b/multipart/samples/iron/Cargo.toml deleted file mode 100644 index c1db1870a..000000000 --- a/multipart/samples/iron/Cargo.toml +++ /dev/null @@ -1,12 +0,0 @@ -[package] -name = "iron" -version = "0.1.0" -authors = ["White-Oak "] - -[dependencies] -iron = "0.2" - -[dependencies.multipart] -version = "0.5.1" -default-features = false -features = ["server", "iron"] diff --git a/multipart/samples/tiny_http/.gitignore b/multipart/samples/tiny_http/.gitignore deleted file mode 100644 index 2f7896d1d..000000000 --- a/multipart/samples/tiny_http/.gitignore +++ /dev/null @@ -1 +0,0 @@ -target/ diff --git a/multipart/samples/tiny_http/Cargo.lock b/multipart/samples/tiny_http/Cargo.lock deleted file mode 100644 index a06ff8547..000000000 --- a/multipart/samples/tiny_http/Cargo.lock +++ /dev/null @@ -1,314 +0,0 @@ -[root] -name = "tiny_http" -version = "0.1.0" -dependencies = [ - "multipart 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", - "tiny_http 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "aho-corasick" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "memchr 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "ascii" -version = "0.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "buf_redux" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "chrono" -version = "0.2.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "num 0.1.31 (registry+https://github.com/rust-lang/crates.io-index)", - "time 0.1.34 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "chunked_transfer" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "encoding" -version = "0.2.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "encoding-index-japanese 1.20141219.5 (registry+https://github.com/rust-lang/crates.io-index)", - "encoding-index-korean 1.20141219.5 (registry+https://github.com/rust-lang/crates.io-index)", - "encoding-index-simpchinese 1.20141219.5 (registry+https://github.com/rust-lang/crates.io-index)", - "encoding-index-singlebyte 1.20141219.5 (registry+https://github.com/rust-lang/crates.io-index)", - "encoding-index-tradchinese 1.20141219.5 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "encoding-index-japanese" -version = "1.20141219.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "encoding_index_tests 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "encoding-index-korean" -version = "1.20141219.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "encoding_index_tests 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "encoding-index-simpchinese" -version = "1.20141219.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "encoding_index_tests 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "encoding-index-singlebyte" -version = "1.20141219.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "encoding_index_tests 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "encoding-index-tradchinese" -version = "1.20141219.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "encoding_index_tests 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "encoding_index_tests" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "env_logger" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "log 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", - "regex 0.1.58 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "kernel32-sys" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "winapi 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "libc" -version = "0.2.8" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "log" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "libc 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "matches" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "memchr" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "libc 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "mime" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "log 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 0.6.15 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "mime_guess" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "mime 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", - "phf 0.7.13 (registry+https://github.com/rust-lang/crates.io-index)", - "phf_codegen 0.7.13 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "multipart" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "buf_redux 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", - "env_logger 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", - "memchr 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", - "mime 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", - "mime_guess 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)", - "rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", - "tempdir 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", - "tiny_http 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "num" -version = "0.1.31" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "phf" -version = "0.7.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "phf_shared 0.7.13 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "phf_codegen" -version = "0.7.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "phf_generator 0.7.13 (registry+https://github.com/rust-lang/crates.io-index)", - "phf_shared 0.7.13 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "phf_generator" -version = "0.7.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "phf_shared 0.7.13 (registry+https://github.com/rust-lang/crates.io-index)", - "rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "phf_shared" -version = "0.7.13" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "rand" -version = "0.3.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "libc 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "regex" -version = "0.1.58" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "aho-corasick 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", - "memchr 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", - "regex-syntax 0.3.0 (registry+https://github.com/rust-lang/crates.io-index)", - "utf8-ranges 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "regex-syntax" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "rustc-serialize" -version = "0.3.18" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "serde" -version = "0.6.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "num 0.1.31 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "tempdir" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "time" -version = "0.1.34" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "kernel32-sys 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "tiny_http" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "ascii 0.5.4 (registry+https://github.com/rust-lang/crates.io-index)", - "chrono 0.2.20 (registry+https://github.com/rust-lang/crates.io-index)", - "chunked_transfer 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", - "encoding 0.2.32 (registry+https://github.com/rust-lang/crates.io-index)", - "rustc-serialize 0.3.18 (registry+https://github.com/rust-lang/crates.io-index)", - "url 0.2.38 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "url" -version = "0.2.38" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "matches 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "rustc-serialize 0.3.18 (registry+https://github.com/rust-lang/crates.io-index)", - "uuid 0.1.18 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "utf8-ranges" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "uuid" -version = "0.1.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "rand 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)", - "rustc-serialize 0.3.18 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "winapi" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "winapi-build" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" - diff --git a/multipart/samples/tiny_http/Cargo.toml b/multipart/samples/tiny_http/Cargo.toml deleted file mode 100644 index fe5bfddfc..000000000 --- a/multipart/samples/tiny_http/Cargo.toml +++ /dev/null @@ -1,12 +0,0 @@ -[package] -name = "tiny_http" -version = "0.1.0" -authors = ["White-Oak "] - -[dependencies] -tiny_http = "0.5" - -[dependencies.multipart] -version = "0.5.1" -default-features = false -features = ["server", "tiny_http"] From 21e86389d9a330a3121c885030a9611dadb48f7c Mon Sep 17 00:00:00 2001 From: Michael Date: Tue, 26 Jul 2016 16:16:22 -0400 Subject: [PATCH 163/453] fix readme --- multipart/examples/README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/multipart/examples/README.md b/multipart/examples/README.md index cc346a06d..66e0a7190 100644 --- a/multipart/examples/README.md +++ b/multipart/examples/README.md @@ -7,7 +7,7 @@ These projects carry the same licenses as [`multipart` itself](https://github.co More sample projects are underway and volunteers to create them are still needed. See [this issue](https://github.com/cybergeek94/multipart/issues/29) for more information. -[`iron`](iron) +[`iron`](iron.rs) ----- Author: [White-Oak][white-oak] @@ -21,9 +21,9 @@ $ cargo run --features "iron" --example iron ``` -[`tiny_http`](tiny_http) +[`tiny_http`](tiny_http.rs) ---------- -Author: [White-Oak](white-oak) +Author: [White-Oak][white-oak] This sample project shows how to use `multipart` with the [`tiny_http` crate](https://crates.io/crates/tiny_http), via `multipart`'s support for the `tiny_http::Request` type. @@ -32,11 +32,11 @@ This sample project shows how to use `multipart` with the [`tiny_http` crate](ht $ cargo run --features "tiny_http" --example tiny_http ``` -[`hyper_server`](hyper_server) ------------------------------ -Author: [Puhrez](puhrez) +[`hyper_server`](hyper_server.rs) +--------------------------------- +Author: [Puhrez][puhrez] -This sample project shows how to use `multipart` with a [`hyper::Server`] (http://hyper.rs/) to intercept multipart requests. +This sample project shows how to use `multipart` with a [`hyper::Server`] (http://hyper.rs/) to intercept multipart requests. ``` $ cargo run --example hyper_server From f5365a56623900c4b67ee22128c8a00b38541b3e Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sat, 30 Jul 2016 16:06:55 -0700 Subject: [PATCH 164/453] Add hyper_client example --- multipart/examples/hyper_client.rs | 36 ++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 multipart/examples/hyper_client.rs diff --git a/multipart/examples/hyper_client.rs b/multipart/examples/hyper_client.rs new file mode 100644 index 000000000..2c8766681 --- /dev/null +++ b/multipart/examples/hyper_client.rs @@ -0,0 +1,36 @@ +extern crate hyper; +extern crate multipart; + +use hyper::client::Request; +use hyper::method::Method; +use hyper::net::Streaming; + +use multipart::client::Multipart; + +fn main() { + let url = "http://localhost:80".parse() + .expect("Failed to parse URL"); + + let request = Request::new(Method::Post, url) + .expect("Failed to create request"); + + let mut multipart = Multipart::from_request(request) + .expect("Failed to create Multipart"); + + write_body(&mut multipart) + .expect("Failed to write multipart body"); + + let _response = multipart.send().expect("Failed to send multipart request"); + + // Optional: read out response +} + +fn write_body(multi: &mut Multipart>) -> hyper::Result<()> { + let mut binary = "Hello world from binary!".as_bytes(); + + try!(multi.write_text("text", "Hello, world!")); + try!(multi.write_file("file", "lorem_ipsum.txt")); + // &[u8] impl Read + multi.write_stream("binary", &mut binary, None, None) + .and(Ok(())) +} \ No newline at end of file From 708af60266700820b50f3e4fb80c86eabd6fbdb9 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sat, 30 Jul 2016 17:07:21 -0700 Subject: [PATCH 165/453] Change trait bound of client::lazy::Multipart::add_file() so that it doesn't need a type annotation --- multipart/src/client/lazy.rs | 37 +++++++++++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/multipart/src/client/lazy.rs b/multipart/src/client/lazy.rs index d677f9c59..8835ca9db 100644 --- a/multipart/src/client/lazy.rs +++ b/multipart/src/client/lazy.rs @@ -5,7 +5,7 @@ use mime::Mime; use std::borrow::Cow; use std::error::Error; use std::fs::File; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::io::prelude::*; use std::{fmt, io, mem}; @@ -131,11 +131,11 @@ impl<'n, 'd> Multipart<'n, 'd> { /// /// ### Note /// Does not check if `path` exists. - pub fn add_file(&mut self, name: N, path: P) -> &mut Self where N: Into>, P: Into> { + pub fn add_file(&mut self, name: N, path: P) -> &mut Self where N: Into>, P: IntoCowPath<'d> { self.fields.push( Field { name: name.into(), - data: Data::File(path.into()), + data: Data::File(path.into_cow_path()), } ); @@ -389,6 +389,37 @@ impl<'d> AsRef<[u8]> for CowStrAsRef<'d> { } } +/// Conversion trait necessary for `Multipart::add_file()` to accept borrowed or owned strings +/// and borrowed or owned paths +pub trait IntoCowPath<'a> { + /// Self-explanatory, hopefully + fn into_cow_path(self) -> Cow<'a, Path>; +} + +impl IntoCowPath<'static> for PathBuf { + fn into_cow_path(self) -> Cow<'static, Path> { + self.into() + } +} + +impl IntoCowPath<'static> for String { + fn into_cow_path(self) -> Cow<'static, Path> { + PathBuf::from(self).into() + } +} + +impl<'a> IntoCowPath<'a> for &'a Path { + fn into_cow_path(self) -> Cow<'a, Path> { + self.into() + } +} + +impl<'a> IntoCowPath<'a> for &'a str { + fn into_cow_path(self) -> Cow<'a, Path> { + Path::new(self).into() + } +} + #[cfg(feature = "hyper")] mod hyper { use hyper::client::{Body, Client, IntoUrl, RequestBuilder, Response}; From 0ec9986787043d8dddcf8508dec48c6675c6dc8b Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sat, 30 Jul 2016 17:31:37 -0700 Subject: [PATCH 166/453] hyper_reqbuilder example --- multipart/examples/hyper_reqbuilder.rs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 multipart/examples/hyper_reqbuilder.rs diff --git a/multipart/examples/hyper_reqbuilder.rs b/multipart/examples/hyper_reqbuilder.rs new file mode 100644 index 000000000..4bfca42ec --- /dev/null +++ b/multipart/examples/hyper_reqbuilder.rs @@ -0,0 +1,18 @@ +extern crate hyper; +extern crate multipart; + +use hyper::Client; + +use multipart::client::lazy::Multipart; + +fn main() { + let mut binary = "Hello world in binary!".as_bytes(); + + let _response = Multipart::new() + .add_text("text", "Hello, world!") + .add_file("file", "lorem_ipsum.txt") + // A little extra type info needed. + .add_stream("binary", &mut binary, None as Option<&str>, None) + .client_request(&Client::new(), "http://localhost:80") + .expect("Error sending multipart request"); +} \ No newline at end of file From 2286a67747fc06aedcea3f2116e10ba2a172db15 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sat, 30 Jul 2016 17:47:56 -0700 Subject: [PATCH 167/453] Add identify impl for IntoCowPath --- multipart/src/client/lazy.rs | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/multipart/src/client/lazy.rs b/multipart/src/client/lazy.rs index 8835ca9db..f964f49d3 100644 --- a/multipart/src/client/lazy.rs +++ b/multipart/src/client/lazy.rs @@ -396,15 +396,15 @@ pub trait IntoCowPath<'a> { fn into_cow_path(self) -> Cow<'a, Path>; } -impl IntoCowPath<'static> for PathBuf { - fn into_cow_path(self) -> Cow<'static, Path> { - self.into() +impl<'a> IntoCowPath<'a> for Cow<'a, Path> { + fn into_cow_path(self) -> Cow<'a, Path> { + self } } -impl IntoCowPath<'static> for String { +impl IntoCowPath<'static> for PathBuf { fn into_cow_path(self) -> Cow<'static, Path> { - PathBuf::from(self).into() + self.into() } } @@ -414,6 +414,12 @@ impl<'a> IntoCowPath<'a> for &'a Path { } } +impl IntoCowPath<'static> for String { + fn into_cow_path(self) -> Cow<'static, Path> { + PathBuf::from(self).into() + } +} + impl<'a> IntoCowPath<'a> for &'a str { fn into_cow_path(self) -> Cow<'a, Path> { Path::new(self).into() From 6d105a93f4a890a650d238062bc1b6581039a55d Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Tue, 2 Aug 2016 18:06:00 -0700 Subject: [PATCH 168/453] Add iron-intercept example --- multipart/examples/iron_intercept.rs | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 multipart/examples/iron_intercept.rs diff --git a/multipart/examples/iron_intercept.rs b/multipart/examples/iron_intercept.rs new file mode 100644 index 000000000..fa328dd6e --- /dev/null +++ b/multipart/examples/iron_intercept.rs @@ -0,0 +1,25 @@ +extern crate iron; +extern crate multipart; + +use iron::prelude::*; + +use multipart::server::Entries; +use multipart::server::iron::Intercept; + +fn main() { + // We start with a basic request handler chain. + let mut chain = Chain::new(|req: &mut Request| + if let Some(entries) = req.extensions.get::() { + Ok(Response::with(format!("{:?}", entries))) + } else { + Ok(Response::with("Not a multipart request")) + } + ); + + // `Intercept` will read out the entries and place them as an extension in the request. + // It has various builder-style methods for changing how it will behave, but has sane settings + // by default. + chain.link_before(Intercept::default()); + + Iron::new(chain).http("localhost:80").unwrap(); +} \ No newline at end of file From a673fd3651fca77812f715d1c4d619fdc3807070 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Tue, 2 Aug 2016 18:06:24 -0700 Subject: [PATCH 169/453] Bump min `tempdir` version, derive `Debug` for `server::SaveDir` --- multipart/Cargo.toml | 4 ++-- multipart/src/server/mod.rs | 14 +------------- 2 files changed, 3 insertions(+), 15 deletions(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index f14a27e9b..3882db6b7 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "multipart" -version = "0.7.0" +version = "0.8.0" authors = ["Austin Bonander "] @@ -21,7 +21,7 @@ log = "0.3" mime = "0.2" mime_guess = "1.6" rand = "0.3" -tempdir = "0.3" +tempdir = ">=0.3.4" [dependencies.buf_redux] optional = true diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index ad75c027e..6f65ac36d 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -659,6 +659,7 @@ impl Entries { } /// The save directory for `Entries`. May be temporary (delete-on-drop) or permanent. +#[derive(Debug)] pub enum SaveDir { /// This directory is temporary and will be deleted, along with its contents, when this wrapper /// is dropped. @@ -738,19 +739,6 @@ impl AsRef for SaveDir { } } -// grrr, no Debug impl for TempDir, can't derive -// FIXME when tempdir > 0.3.4 is released (Debug PR landed 3/3/2016) -impl fmt::Debug for SaveDir { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - use self::SaveDir::*; - - match *self { - Temp(ref tempdir) => write!(f, "SaveDir::Temp({:?})", tempdir.path()), - Perm(ref path) => write!(f, "SaveDir::Perm({:?})", path), - } - } -} - /// A file saved to the local filesystem from a multipart request. #[derive(Debug)] pub struct SavedFile { From 104eba20b40a0fb9110217b24696feb106231186 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Tue, 2 Aug 2016 18:38:47 -0700 Subject: [PATCH 170/453] Add comment to hyper_reqbuilder example --- multipart/examples/hyper_reqbuilder.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/multipart/examples/hyper_reqbuilder.rs b/multipart/examples/hyper_reqbuilder.rs index 4bfca42ec..05c5539c8 100644 --- a/multipart/examples/hyper_reqbuilder.rs +++ b/multipart/examples/hyper_reqbuilder.rs @@ -13,6 +13,7 @@ fn main() { .add_file("file", "lorem_ipsum.txt") // A little extra type info needed. .add_stream("binary", &mut binary, None as Option<&str>, None) + // Request is sent here .client_request(&Client::new(), "http://localhost:80") .expect("Error sending multipart request"); } \ No newline at end of file From 6079982458b80e07a64706c7d9f4adbd6a50dd78 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Fri, 5 Aug 2016 16:33:09 -0700 Subject: [PATCH 171/453] Update and reformat `examples/README.md` --- multipart/examples/README.md | 76 +++++++++++++++++++++++++++--------- 1 file changed, 57 insertions(+), 19 deletions(-) diff --git a/multipart/examples/README.md b/multipart/examples/README.md index 66e0a7190..e17d9a9f0 100644 --- a/multipart/examples/README.md +++ b/multipart/examples/README.md @@ -1,47 +1,85 @@ -`multipart` Sample Projects +`multipart` Examples =========================== -These are simple but fully fledged Cargo/Rust application projects to show how to use `multipart` with the various crates it integrates with. +These example files show how to use `multipart` with the various crates it integrates with. -These projects carry the same licenses as [`multipart` itself](https://github.com/cybergeek94/multipart#license), though this may be lightened to a copyright-free license in the near future. +These files carry the same licenses as [`multipart` itself](https://github.com/cybergeek94/multipart#license), though this may be lightened to a copyright-free license in the near future. -More sample projects are underway and volunteers to create them are still needed. See [this issue](https://github.com/cybergeek94/multipart/issues/29) for more information. +More examples are underway and volunteers to create them are still needed. See [this issue](https://github.com/cybergeek94/multipart/issues/29) for more information. -[`iron`](iron.rs) ------ -Author: [White-Oak][white-oak] +##Client -This sample project shows how to use `multipart` with the [Iron web application framework](http://ironframework.io/), via `multipart`'s support -for the `iron::Request` type. +Examples for the client-side integrations of `multipart`'s API. -To run: +[`hyper_client`](hyper_client.rs) +--------------------------------- +Author: [cybergeek94][cybergeek94] + +This example showcases usage of `multipart` with the `hyper::client::Request` API. ``` -$ cargo run --features "iron" --example iron +$ cargo run --example hyper_client ``` +[`hyper_reqbuilder`](hyper_reqbuilder.rs) +----------------------------------------- +Author: [cybergeek94][cybergeek94] -[`tiny_http`](tiny_http.rs) ----------- -Author: [White-Oak][white-oak] - -This sample project shows how to use `multipart` with the [`tiny_http` crate](https://crates.io/crates/tiny_http), via `multipart`'s support for the `tiny_http::Request` type. - +This example showcases usage of `multipart` with Hyper's new `Client` API, +via the lazy-writing capabilities of `multipart::client::lazy`. ``` -$ cargo run --features "tiny_http" --example tiny_http +$ cargo run --example hyper_reqbuilder ``` + +##Server + [`hyper_server`](hyper_server.rs) --------------------------------- Author: [Puhrez][puhrez] -This sample project shows how to use `multipart` with a [`hyper::Server`] (http://hyper.rs/) to intercept multipart requests. +This example shows how to use `multipart` with a [`hyper::Server`] (http://hyper.rs/) to intercept multipart requests. ``` $ cargo run --example hyper_server ``` +[`iron`](iron.rs) +----------------- +Author: [White-Oak][white-oak] + +This example shows how to use `multipart` with the [Iron web application framework](http://ironframework.io/), via `multipart`'s support +for the `iron::Request` type. + +To run: + +``` +$ cargo run --features iron --example iron +``` + +[`iron_intercept`](iron_intercept.rs) +------------------------------------- +Author: [cybergeek94][cybergeek94] + +This example shows how to use `multipart`'s specialized `Intercept` middleware with Iron, which reads out all fields and +files to local storage so they can be accessed arbitrarily. + +``` +$ cargo run --features iron --example iron_intercept +``` + +[`tiny_http`](tiny_http.rs) +--------------------------- +Author: [White-Oak][white-oak] + +This example shows how to use `multipart` with the [`tiny_http` crate](https://crates.io/crates/tiny_http), via `multipart`'s support for the `tiny_http::Request` type. + +``` +$ cargo run --features tiny_http --example tiny_http +``` [puhrez]: https://github.com/puhrez [white-oak]: https://github.com/white-oak +[cybergeek94]: https://github.com/cybergeek94 + From ee206b199ad59d756019665cb6dd2b4ad046a46c Mon Sep 17 00:00:00 2001 From: Sebastian Blei Date: Mon, 8 Aug 2016 16:18:11 +0200 Subject: [PATCH 172/453] Added nickel.rs example --- multipart/examples/nickel.rs | 75 ++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 multipart/examples/nickel.rs diff --git a/multipart/examples/nickel.rs b/multipart/examples/nickel.rs new file mode 100644 index 000000000..fc7637363 --- /dev/null +++ b/multipart/examples/nickel.rs @@ -0,0 +1,75 @@ +#[macro_use] +extern crate nickel; +extern crate multipart; + +use std::fs::File; +use std::io::Read; +use multipart::server::{Entries, Multipart, SaveResult}; +use nickel::{HttpRouter, MiddlewareResult, Nickel, Request, Response}; + +fn handle_multipart<'mw>(req: &mut Request, mut res: Response<'mw>) -> MiddlewareResult<'mw> { + match Multipart::from_request(req) { + Ok(mut multipart) => { + match multipart.save_all() { + SaveResult::Full(entries) => process_entries(res, entries), + + SaveResult::Partial(entries, e) => { + println!("Partial errors ... {:?}", e); + return process_entries(res, entries); + }, + + SaveResult::Error(e) => { + println!("There are errors in multipart POSTing ... {:?}", e); + res.set(nickel::status::StatusCode::InternalServerError); + return res.send(format!("Server could not handle multipart POST! {:?}", e)); + }, + } + } + Err(_) => { + res.set(nickel::status::StatusCode::BadRequest); + return res.send("Request seems not was a multipart request") + } + } +} + +/// Processes saved entries from multipart request. +/// Returns an OK response or an error. +fn process_entries<'mw>(res: Response<'mw>, entries: Entries) -> MiddlewareResult<'mw> { + for (name, field) in entries.fields { + println!(r#"Field "{}": "{}""#, name, field); + } + + for (name, savedfile) in entries.files { + let filename = match savedfile.filename { + Some(s) => s, + None => "None".into(), + }; + + match File::open(savedfile.path) { + Ok(mut file) => { + let mut contents = String::new(); + match file.read_to_string(&mut contents) { + Ok(sz) => println!("File: \"{}\" is of size: {}b.", filename, sz), + Err(e) => println!("Could not read file's \"{}\" size. Error: {:?}", filename, e), + } + println!(r#"Field "{}" is file "{}":"#, name, filename); + println!("{}", contents); + file + } + Err(e) => { + println!("Could open file \"{}\". Error: {:?}", filename, e); + return res.error(nickel::status::StatusCode::BadRequest, "The uploaded file was not readable") + } + }; + } + + res.send("Ok") +} + +fn main() { + let mut srv = Nickel::new(); + + srv.post("/multipart_upload/", handle_multipart); + + srv.listen("127.0.0.1:6868"); +} From 14e2c788de94dadb5548f28227c4410af0f1d432 Mon Sep 17 00:00:00 2001 From: Sebastian Blei Date: Mon, 8 Aug 2016 16:29:23 +0200 Subject: [PATCH 173/453] Added nickel.rs example documentation --- multipart/examples/README.md | 11 +++++++++++ multipart/examples/nickel.rs | 8 ++++++++ 2 files changed, 19 insertions(+) diff --git a/multipart/examples/README.md b/multipart/examples/README.md index 66e0a7190..084fa7412 100644 --- a/multipart/examples/README.md +++ b/multipart/examples/README.md @@ -42,6 +42,17 @@ This sample project shows how to use `multipart` with a [`hyper::Server`] (http: $ cargo run --example hyper_server ``` +[`nickel`](nickel.rs) +--------------------------------- +Author: [iamsebastian][iamsebastian] + +How you could use this multipart crate to handle multipart uploads in [nickel.rs](https://nickel.rs). + +``` +$ cargo run --example nickel --features nickel +``` + +[iamsebastian]: https://github.com/iamsebastian [puhrez]: https://github.com/puhrez [white-oak]: https://github.com/white-oak diff --git a/multipart/examples/nickel.rs b/multipart/examples/nickel.rs index fc7637363..647257ce7 100644 --- a/multipart/examples/nickel.rs +++ b/multipart/examples/nickel.rs @@ -71,5 +71,13 @@ fn main() { srv.post("/multipart_upload/", handle_multipart); + // Start this example via: + // + // `cargo run --example nickel --features nickel` + // + // And - if you are in the root of this repository - do an example + // upload via: + // + // `curl -F file=@LICENSE 'http://localhost:6868/multipart_upload/'` srv.listen("127.0.0.1:6868"); } From dc1dd2db426ce86dbea22f5a2b87b3dfba7c74db Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Mon, 15 Aug 2016 19:21:16 -0700 Subject: [PATCH 174/453] Update Cargo.toml --- multipart/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index f14a27e9b..3c9b115fa 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -9,9 +9,9 @@ description = "A backend-agnostic extension for HTTP libraries that provides sup keywords = ["form-data", "hyper", "iron", "http", "upload"] -repository = "http://github.com/cybergeek94/multipart" +repository = "http://github.com/abonander/multipart" -documentation = "http://cybergeek94.github.io/multipart/doc/multipart/index.html" +documentation = "http://abonander.github.io/multipart/doc/multipart/index.html" license = "MIT OR Apache-2.0" From 2a2faa9e65942bdf9cc083e79114d33899efb8c2 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Mon, 15 Aug 2016 19:21:43 -0700 Subject: [PATCH 175/453] Update README.md --- multipart/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/multipart/README.md b/multipart/README.md index 943e010b9..170b4d8dd 100644 --- a/multipart/README.md +++ b/multipart/README.md @@ -1,11 +1,11 @@ -Multipart [![Build Status](https://travis-ci.org/cybergeek94/multipart.svg?branch=master)](https://travis-ci.org/cybergeek94/multipart) [![On Crates.io](https://img.shields.io/crates/v/multipart.svg)](https://crates.io/crates/multipart) +Multipart [![Build Status](https://travis-ci.org/abonander/multipart.svg?branch=master)](https://travis-ci.org/abonander/multipart) [![On Crates.io](https://img.shields.io/crates/v/multipart.svg)](https://crates.io/crates/multipart) ========= Client- and server-side abstractions for HTTP file uploads (POST requests with `Content-Type: multipart/form-data`). Supports several different HTTP crates. -###[Documentation](http://cybergeek94.github.io/multipart/doc/multipart/index.html) +###[Documentation](http://abonander.github.io/multipart/doc/multipart/index.html) ##Integrations From ba1e39b584fa2ddddab268223b3fe7383967c3f7 Mon Sep 17 00:00:00 2001 From: mbme Date: Thu, 1 Sep 2016 19:45:39 +0300 Subject: [PATCH 176/453] Update iron dependency version --- multipart/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 3c9b115fa..6c70b866a 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -33,7 +33,7 @@ version = "0.8" [dependencies.iron] optional = true -version = "0.3" +version = "0.4" [dependencies.memchr] optional = true From 495930530d4720f271e96fdbe539eed4d16f54c0 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Thu, 22 Sep 2016 15:49:39 -0700 Subject: [PATCH 177/453] Upgrade dependencies, move env_logger to test config only --- multipart/Cargo.toml | 14 ++++++++------ multipart/src/lib.rs | 3 ++- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 3882db6b7..0d14359cc 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -16,24 +16,23 @@ documentation = "http://cybergeek94.github.io/multipart/doc/multipart/index.html license = "MIT OR Apache-2.0" [dependencies] -env_logger = "0.3" log = "0.3" mime = "0.2" -mime_guess = "1.6" +mime_guess = "1.8" rand = "0.3" tempdir = ">=0.3.4" [dependencies.buf_redux] optional = true -version = "0.1" +version = "0.3" [dependencies.hyper] optional = true -version = "0.8" +version = "0.9" [dependencies.iron] optional = true -version = "0.3" +version = "0.4" [dependencies.memchr] optional = true @@ -42,12 +41,15 @@ version = "0.1" # NOTE: use `nickel_` feature, as `hyper` feature is required. [dependencies.nickel] optional = true -version = "0.8" +version = "0.9" [dependencies.tiny_http] optional = true version = "0.5" +[dev-dependencies] +env_logger = "0.3" + [features] client = [] default = ["hyper", "server", "client"] diff --git a/multipart/src/lib.rs b/multipart/src/lib.rs index 1815bcd0c..018ed9c47 100644 --- a/multipart/src/lib.rs +++ b/multipart/src/lib.rs @@ -28,7 +28,8 @@ //! feature. #![warn(missing_docs)] #[macro_use] extern crate log; -extern crate env_logger; + +#[cfg(test)] extern crate env_logger; extern crate mime; extern crate mime_guess; From f2677ae3bd7af9fe41e01d76e9734f3d1ad6a5dd Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Thu, 22 Sep 2016 16:01:34 -0700 Subject: [PATCH 178/453] Update repo and doc link --- multipart/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 0d14359cc..e3c23fe4a 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -9,9 +9,9 @@ description = "A backend-agnostic extension for HTTP libraries that provides sup keywords = ["form-data", "hyper", "iron", "http", "upload"] -repository = "http://github.com/cybergeek94/multipart" +repository = "http://github.com/abonander/multipart" -documentation = "http://cybergeek94.github.io/multipart/doc/multipart/index.html" +documentation = "http://abonander.github.io/multipart/doc/multipart/index.html" license = "MIT OR Apache-2.0" From f5c65416b6ed19b431813491439fd22c9cf06623 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Thu, 22 Sep 2016 16:20:15 -0700 Subject: [PATCH 179/453] Change links in README --- multipart/examples/README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/multipart/examples/README.md b/multipart/examples/README.md index d3a68935b..680064d1e 100644 --- a/multipart/examples/README.md +++ b/multipart/examples/README.md @@ -3,9 +3,9 @@ These example files show how to use `multipart` with the various crates it integrates with. -These files carry the same licenses as [`multipart` itself](https://github.com/cybergeek94/multipart#license), though this may be lightened to a copyright-free license in the near future. +These files carry the same licenses as [`multipart` itself](https://github.com/abonander/multipart#license), though this may be lightened to a copyright-free license in the near future. -More examples are underway and volunteers to create them are still needed. See [this issue](https://github.com/cybergeek94/multipart/issues/29) for more information. +More examples are underway and volunteers to create them are still needed. See [this issue](https://github.com/abonander/multipart/issues/29) for more information. ##Client @@ -13,7 +13,7 @@ Examples for the client-side integrations of `multipart`'s API. [`hyper_client`](hyper_client.rs) --------------------------------- -Author: [cybergeek94][cybergeek94] +Author: [abonander][abonander] This example showcases usage of `multipart` with the `hyper::client::Request` API. @@ -23,7 +23,7 @@ $ cargo run --example hyper_client [`hyper_reqbuilder`](hyper_reqbuilder.rs) ----------------------------------------- -Author: [cybergeek94][cybergeek94] +Author: [abonander][abonander] This example showcases usage of `multipart` with Hyper's new `Client` API, via the lazy-writing capabilities of `multipart::client::lazy`. @@ -60,7 +60,7 @@ $ cargo run --features iron --example iron [`iron_intercept`](iron_intercept.rs) ------------------------------------- -Author: [cybergeek94][cybergeek94] +Author: [abonander][abonander] This example shows how to use `multipart`'s specialized `Intercept` middleware with Iron, which reads out all fields and files to local storage so they can be accessed arbitrarily. @@ -103,5 +103,5 @@ $ cargo run --example nickel --features nickel [iamsebastian]: https://github.com/iamsebastian [puhrez]: https://github.com/puhrez [white-oak]: https://github.com/white-oak -[cybergeek94]: https://github.com/cybergeek94 +[abonander]: https://github.com/abonander From ca417c5abba498d34248d4e73bec1e5186e04841 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Thu, 22 Sep 2016 18:06:55 -0700 Subject: [PATCH 180/453] Remove unused import --- multipart/src/server/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index 6f65ac36d..04ef6f23d 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -20,7 +20,7 @@ use std::collections::HashMap; use std::fs::{self, File}; use std::io::prelude::*; use std::path::{Path, PathBuf}; -use std::{fmt, io, mem, ptr}; +use std::{io, mem, ptr}; use self::boundary::BoundaryReader; From d60182e7f321035011189092f58dae42a55e6df3 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Thu, 22 Sep 2016 18:07:35 -0700 Subject: [PATCH 181/453] Fix unused result in nickel example --- multipart/examples/nickel.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/examples/nickel.rs b/multipart/examples/nickel.rs index 647257ce7..0da452e48 100644 --- a/multipart/examples/nickel.rs +++ b/multipart/examples/nickel.rs @@ -79,5 +79,5 @@ fn main() { // upload via: // // `curl -F file=@LICENSE 'http://localhost:6868/multipart_upload/'` - srv.listen("127.0.0.1:6868"); + srv.listen("127.0.0.1:6868").expect("Failed to bind server"); } From 4ab29929696a09a84ad549af26c5caab2eee53f3 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Fri, 23 Sep 2016 00:44:19 -0700 Subject: [PATCH 182/453] Update READMEs --- multipart/README.md | 2 +- multipart/examples/README.md | 7 ++----- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/multipart/README.md b/multipart/README.md index 170b4d8dd..4b0072b43 100644 --- a/multipart/README.md +++ b/multipart/README.md @@ -9,7 +9,7 @@ Supports several different HTTP crates. ##Integrations -Sample projects demonstrating how to use `multipart` with these crates are available under [`samples/`](samples). +Example files demonstrating how to use `multipart` with these crates are available under [`examples/`](examples). ####[Hyper](http://hyper.rs) via the `hyper` feature (enabled by default). diff --git a/multipart/examples/README.md b/multipart/examples/README.md index 680064d1e..00294c9c5 100644 --- a/multipart/examples/README.md +++ b/multipart/examples/README.md @@ -5,8 +5,6 @@ These example files show how to use `multipart` with the various crates it integ These files carry the same licenses as [`multipart` itself](https://github.com/abonander/multipart#license), though this may be lightened to a copyright-free license in the near future. -More examples are underway and volunteers to create them are still needed. See [this issue](https://github.com/abonander/multipart/issues/29) for more information. - ##Client Examples for the client-side integrations of `multipart`'s API. @@ -32,7 +30,6 @@ via the lazy-writing capabilities of `multipart::client::lazy`. $ cargo run --example hyper_reqbuilder ``` - ##Server [`hyper_server`](hyper_server.rs) @@ -83,7 +80,7 @@ $ cargo run --features tiny_http --example tiny_http --------------------------------- Author: [Puhrez][puhrez] -This sample project shows how to use `multipart` with a [`hyper::Server`] (http://hyper.rs/) to intercept multipart requests. +This example shows how to use `multipart` with a [`hyper::Server`] (http://hyper.rs/) to intercept multipart requests. ``` $ cargo run --example hyper_server @@ -93,7 +90,7 @@ $ cargo run --example hyper_server --------------------------------- Author: [iamsebastian][iamsebastian] -How you could use this multipart crate to handle multipart uploads in [nickel.rs](https://nickel.rs). +This example shows how to use `multipart` to handle multipart uploads in [nickel.rs](https://nickel.rs). ``` $ cargo run --example nickel --features nickel From 46a6fe42a5115a7c1ce63af2057fec5662dc0326 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sat, 24 Sep 2016 02:18:31 -0700 Subject: [PATCH 183/453] Use docs.rs doc hosting; add .idea/ to .gitignore --- multipart/.gitignore | 1 + multipart/.travis.yml | 8 ++------ multipart/Cargo.toml | 2 +- multipart/README.md | 2 +- multipart/deploy-docs.sh | 24 ------------------------ multipart/id_rsa.enc | Bin 3248 -> 0 bytes 6 files changed, 5 insertions(+), 32 deletions(-) delete mode 100644 multipart/deploy-docs.sh delete mode 100644 multipart/id_rsa.enc diff --git a/multipart/.gitignore b/multipart/.gitignore index f79cd6203..f55131aa6 100644 --- a/multipart/.gitignore +++ b/multipart/.gitignore @@ -14,4 +14,5 @@ # Generated by Cargo /target/ +.idea/ diff --git a/multipart/.travis.yml b/multipart/.travis.yml index 3bbfc1dc7..490a64c65 100644 --- a/multipart/.travis.yml +++ b/multipart/.travis.yml @@ -1,4 +1,6 @@ language: rust +cache: cargo +sudo: false rust: - stable - beta @@ -10,9 +12,3 @@ script: - cargo build -v --features all - cargo test -v --features all - cargo doc -v --no-deps --features all -after_success: - - | - test ${TRAVIS_PULL_REQUEST} == "false" && \ - test "${TRAVIS_BUILD_NUMBER}.1" == "${TRAVIS_JOB_NUMBER}" && \ - test -n ${TRAVIS_TAG} && \ - bash deploy-docs.sh diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index e3c23fe4a..3b3c1acab 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -11,7 +11,7 @@ keywords = ["form-data", "hyper", "iron", "http", "upload"] repository = "http://github.com/abonander/multipart" -documentation = "http://abonander.github.io/multipart/doc/multipart/index.html" +documentation = "http://docs.rs/multipart/" license = "MIT OR Apache-2.0" diff --git a/multipart/README.md b/multipart/README.md index 4b0072b43..92763cbcd 100644 --- a/multipart/README.md +++ b/multipart/README.md @@ -5,7 +5,7 @@ Client- and server-side abstractions for HTTP file uploads (POST requests with Supports several different HTTP crates. -###[Documentation](http://abonander.github.io/multipart/doc/multipart/index.html) +###[Documentation](http://docs.rs/multipart/) ##Integrations diff --git a/multipart/deploy-docs.sh b/multipart/deploy-docs.sh deleted file mode 100644 index 638c1530f..000000000 --- a/multipart/deploy-docs.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/sh - - -#Decrypt RSA key -mkdir -p ~/.ssh -openssl aes-256-cbc -K $encrypted_67079d13adee_key -iv $encrypted_67079d13adee_iv -in id_rsa.enc -out ~/.ssh/id_rsa -d -chmod 600 ~/.ssh/id_rsa - -git config user.name "multipart doc upload" -git config user.email "nobody@example.com" - -git checkout --orphan gh-pages - -git reset -git clean -d -x -f -e target - -cp -R target/doc . -rm -rf target - -git add -A - -git commit -qm "Documentation for ${TRAVIS_TAG}" -git remote set-url origin git@github.com:cybergeek94/multipart.git -git push -f origin gh-pages diff --git a/multipart/id_rsa.enc b/multipart/id_rsa.enc deleted file mode 100644 index 57b91eb9a27d5607d18e041ad2be203381dfbb70..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3248 zcmV;h3{Ug<(--whIQygTHqySZ8fQ*C9@Bayb&6J2%?dIGbz8szkt2>AShqfiBd~G= zz9=j^ALCvX$&A8OVV79?8(AgCO)FRE~>13?Aqc^d_*yXa+h5w-k6Tj#K@r{r1z zyYSjoi#}n1mvmjo;vm)OFRTFDK?Jn@s#!5NxdW#TpZE2_v*yoMmHc}8fWi|&ITZ*| zi#rd44IJaA zwMt}^1Oo`@jU|}v0F*ODAl>_H=U%u_21 z!rQK{h_G1)pQ?yQo{aJYtQP)uYHOjyRNFIpi*ymuomw=^h=&Z4ux9z-vXpi)C$T}Y zWH3zu^OfnskAIG0z5BzVDmq_&O50c_A#0+KWF*6c1_UFuODJQ%_wAiiBvHPrAhD!* za^r&>z~qT3W_OQ0+Hr;IPoL%+rLNT#Ee~Ww^WX0pNo!!RebWGI-XX%1-4Y* zdX%_1;MAvBT#+!$)5zuCU*qE?Bv9%r8(KKBT$Wy0;xjb$YtCyko$8>#?Fg=ihjJ?0 zAq3#~AK2puZo|w*ql~_Nzf%>1v;04{!f1Di%MCCl8o(renB% z{N$4p0ewM?j~UJB`m1uHh04U~4z9vu<7F?@Z^?aAP82iYNcx}ZC+2zhR%9_%6C1Hg z;4Sm^1$^qvQXv!k&L{>c$Ecv0yq1}xBq7ej5oIOY!@e!7U)284))uV+5*km*;BaS zhiRcMX|fTzAZ?74@fpxrbnF=eQ?>{8@&mcpEEFO|(bT%!xGsF1@^dGF*_0;*0;UxB zA742eKBp=w*L^=gR4;fQDtX)ktKYJ(kjidJFxv5hG03^>K2LH4pl&WxqsN^8DC$Z# z5YdwyK|3p?P}6Ogl&e@#1?#>{rd9VgEud7YpI*J^(K>G#org@qCPa9TI{Gz)&xN{LL~F;eQE^f%E!HU9vPz< zk@^ZHnVDV1M>O(DSmYUQ?l#gi7Tx=dveH{n^n3=zJ<%bb3|8cWczPpG=>XrUH3?hv z-jcK@QjjA^=JT0!Za!bxBMbTfbI?u)e7@hpijazEFQ6O3Vc@Oc&DL(YlSv#3SpO$A zC9=I#W8sfN!8fDK`AR&zJ6ba%)^PjPxHu!WRZ9805N{TJvn%tzcu;!J1&{Ql+I`J# z7LhyyFDuGZzj{XAjx1lY!oSPlRl*Eg17A^DIQS+=hM3K{U@B79`<4@Opl$Qq$_NYH zyyvchxvDoB!ONYkAO-WmR-oi)gYuc*iW#NvGMalp_+LG8Tac^IqhND;W}y}(GK&_N z%piA5E(XgPmBfAOvzA7`@R?=6t;%kB>JwC9#IGRn0!x-Vx+ z0u;{krJIP{i;OjeaVoCboOTUd78e*by(V@tgg;QzyBWqbC$l|9xV!=O>a&hS^7C&# z#^T2f;w!J4xX(b0Q4&HUX5hHK+`Ql@|Mfc{Z#e5WOJNHjqd>yoeCxzZxW=)5l7vps zkpcJe=5dunK?%sO6wF_%8^4205px&mPV|8onaE!?eY2oJo!&B-rjJTFrh_AZPu>>( z3%+>3Ee`AqQDo%|S834ItCeo61XfMuP-4rz(Fi#UC&2BL!zi-sU}6VCoY5!*tXbQ% zdv%p0F$9fMA|)f{t<_*Z9dGh2fWf5)+tl6bw6zm?F3-rDvqdfk(|JyNZ#08`loeme*N}y*2Jf>2(>W~*512Ks zWk^>?;dj7ikvldi6p;2E4I&;O%L9ClwXfnig#niQ@ZRcs5jSZT*b-2U^6C$Oot}RY zj0);FQX>MF8~rkgQM^9AUrDLqq|aY`kFytHjpO9O6#{A5jaQ|%=K|L9A;vCHLpPcI zE~{U|3Wq<<+DxJxQa(cL7Eznc$~3~-Nf#E5D5aOyfRLpRrKD+Qk11;<>-N?DrBmT8(0)MjqAvN+<5J(+gr) zRPu7u{Oq=aVgUe%bH^~~!!^|Qvi4iqk=?7GMcw>eaYs;x zQc@l$YHk&yu8#7LZ|>fAs-Ub((aOMXulr_X3FY#|9pZFYjLJ+ucp3slv$UFz zL}!B3PZ#w5~37I40cA&rwve9|RF{g{Ni!b5~olAqxWruUrRmW8&Upn8^5EkYRss)OQEtwT6|wAmvlxIg!rFziJuKFV>tOMSO?LZ@!VtNLO4P zo_&ixn09NEPO5|h^#q-l(oDJ6k;cxoGXJ&(-oXidKs>P-krr59)ft1CRn?`G;Io#je5PyP5X|hYxUCXi zYT z+lWhyNt+)@+r0#6?`%4~tl5!_m22$Z!qy6yoVBG~VM6c}+~!S2WPa4YHB_*Za%adV zpb9<+b;ZdP0%-%u_-)@StnA3(7YtK9Fi?GDDHiv2sLG?S{}roE0P{dmlIjWUJ0=XE zSZ1hI$+PSFkF7(@2J+RhOmd)Y#y1FQqDq(B&yx2T^@R9?k{Bq?^Wky8m?jl$Sjj9A zO=nsaARhT=2>o{z7S83o_b>oYy%_Q?JwEpm9Di)J%r7&dHb)er1`O)iuYAC0!C!E1 zC>5d=8o84N+v;a|uH&0d>k~MBX1|mAkfJT3HzmP%6&XpO%JPY}F(yk#!2?i?0ps3t z^+Pfy<#!M1Y)>^pOlrCJC^YFX_+>$ ibOz?PRC&r%f0n>{-?UQNIbfGrs=8oKVl)CV^q5Y13usXQ From 3482f06ce172c13ea9f85800c6cb7720b8f6e925 Mon Sep 17 00:00:00 2001 From: Laurent Arnoud Date: Sat, 19 Nov 2016 16:47:03 +0100 Subject: [PATCH 184/453] Disable hyper default features for openssl deps This crates does not need default ssl feature from hyper Disable it allow to compile this crate with deps having different openssl version from hyper 0.9 --- multipart/Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 3b3c1acab..54815c93e 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -28,6 +28,7 @@ version = "0.3" [dependencies.hyper] optional = true +default-features = false version = "0.9" [dependencies.iron] From d1d31340325b3f5e8ab4d7384a6546ffce954fbf Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sat, 24 Sep 2016 02:20:21 -0700 Subject: [PATCH 185/453] bump cargo version --- multipart/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 54815c93e..47da72550 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "multipart" -version = "0.8.0" +version = "0.8.1" authors = ["Austin Bonander "] From 0e7d249a53df66728763a86dcd1b727e658d4008 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Mon, 17 Oct 2016 16:23:07 -0700 Subject: [PATCH 186/453] Add `httparse` --- multipart/Cargo.toml | 8 +- multipart/src/server/boundary.rs | 2 +- multipart/src/server/mod.rs | 125 ++++-------------- multipart/src/server/parse.rs | 220 +++++++++++++++++++++++++++++++ 4 files changed, 250 insertions(+), 105 deletions(-) create mode 100644 multipart/src/server/parse.rs diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 47da72550..164bc6e81 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "multipart" -version = "0.8.1" +version = "0.9.0-alpha.1" authors = ["Austin Bonander "] @@ -26,6 +26,10 @@ tempdir = ">=0.3.4" optional = true version = "0.3" +[dependencies.httparse] +optional = true +version = "1.1" + [dependencies.hyper] optional = true default-features = false @@ -54,6 +58,6 @@ env_logger = "0.3" [features] client = [] default = ["hyper", "server", "client"] -server = ["buf_redux", "memchr"] +server = ["buf_redux", "memchr", "httparse"] nickel_ = ["nickel", "hyper"] all = ["iron", "nickel_", "tiny_http"] diff --git a/multipart/src/server/boundary.rs b/multipart/src/server/boundary.rs index 1e1f4235d..0f842fbe2 100644 --- a/multipart/src/server/boundary.rs +++ b/multipart/src/server/boundary.rs @@ -41,7 +41,7 @@ impl BoundaryReader where R: Read { fn read_to_boundary(&mut self) -> io::Result<&[u8]> { use log::LogLevel; - let buf = try!(fill_buf_min(&mut self.buf, self.boundary.len())); + let buf = try!(fill_buf_min(&mut self.buf, self.boundary.len() * 2)); if log_enabled!(LogLevel::Trace) { trace!("Buf: {:?}", String::from_utf8_lossy(buf)); diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index 04ef6f23d..31cb79ff2 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -11,6 +11,8 @@ //! to accept, parse, and serve HTTP `multipart/form-data` requests (file uploads). //! //! See the `Multipart` struct for more info. + + use mime::Mime; use tempdir::TempDir; @@ -23,6 +25,7 @@ use std::path::{Path, PathBuf}; use std::{io, mem, ptr}; use self::boundary::BoundaryReader; +use self::parse::MultipartHeaders; macro_rules! try_opt ( ($expr:expr) => ( @@ -30,11 +33,22 @@ macro_rules! try_opt ( Some(val) => val, None => return None, } + ); + ($expr:expr, $before_ret:expr) => ( + match $expr { + Some(val) => val, + None => { + $before_ret; + return None; + } + } ) ); mod boundary; +mod parse; + #[cfg(feature = "hyper")] pub mod hyper; @@ -100,9 +114,8 @@ impl Multipart { MultipartField::read_from(self) } - fn read_content_disposition(&mut self) -> io::Result> { - let line = try!(self.read_line()); - Ok(ContentDisp::read_from(line)) + fn read_field_headers(&mut self) -> io::Result> { + MultipartHeaders::parse(&mut self.source) } /// Call `f` for each entry in the multipart request. @@ -121,12 +134,6 @@ impl Multipart { } } - fn read_content_type(&mut self) -> io::Result> { - debug!("Read content type!"); - let line = try!(self.read_line()); - Ok(ContentType::read_from(line)) - } - /// Read the request fully, parsing all fields and saving all files in a new temporary /// directory under the OS temporary directory. /// @@ -309,92 +316,6 @@ impl SaveResult { } } -struct ContentType { - val: Mime, - #[allow(dead_code)] - boundary: Option, -} - -impl ContentType { - fn read_from(line: &str) -> Option { - const CONTENT_TYPE: &'static str = "Content-Type:"; - const BOUNDARY: &'static str = "boundary=\""; - - debug!("Reading Content-Type header from line: {:?}", line); - - if let Some((cont_type, after_cont_type)) = get_str_after(CONTENT_TYPE, ';', line) { - let content_type = read_content_type(cont_type.trim()); - - let boundary = get_str_after(BOUNDARY, '"', after_cont_type).map(|tup| tup.0.into()); - - Some(ContentType { - val: content_type, - boundary: boundary, - }) - } else { - get_remainder_after(CONTENT_TYPE, line).map(|cont_type| { - let content_type = read_content_type(cont_type.trim()); - ContentType { val: content_type, boundary: None } - }) - } - } -} - -fn read_content_type(cont_type: &str) -> Mime { - cont_type.parse().ok().unwrap_or_else(::mime_guess::octet_stream) -} - -struct ContentDisp { - field_name: String, - filename: Option, -} - -impl ContentDisp { - fn read_from(line: &str) -> Option { - debug!("Reading Content-Disposition from line: {:?}", line); - - if line.is_empty() { - return None; - } - - const CONT_DISP: &'static str = "Content-Disposition:"; - const NAME: &'static str = "name=\""; - const FILENAME: &'static str = "filename=\""; - - let after_disp_type = { - let (disp_type, after_disp_type) = try_opt!(get_str_after(CONT_DISP, ';', line)); - let disp_type = disp_type.trim(); - - if disp_type != "form-data" { - error!("Unexpected Content-Disposition value: {:?}", disp_type); - return None; - } - - after_disp_type - }; - - let (field_name, after_field_name) = try_opt!(get_str_after(NAME, '"', after_disp_type)); - - let filename = get_str_after(FILENAME, '"', after_field_name) - .map(|(filename, _)| filename.to_owned()); - - Some(ContentDisp { field_name: field_name.to_owned(), filename: filename }) - } -} - -/// Get the string after `needle` in `haystack`, stopping before `end_val_delim` -fn get_str_after<'a>(needle: &str, end_val_delim: char, haystack: &'a str) -> Option<(&'a str, &'a str)> { - let val_start_idx = try_opt!(haystack.find(needle)) + needle.len(); - let val_end_idx = try_opt!(haystack[val_start_idx..].find(end_val_delim)) + val_start_idx; - Some((&haystack[val_start_idx..val_end_idx], &haystack[val_end_idx..])) -} - -/// Get everything after `needle` in `haystack` -fn get_remainder_after<'a>(needle: &str, haystack: &'a str) -> Option<(&'a str)> { - let val_start_idx = try_opt!(haystack.find(needle)) + needle.len(); - Some(&haystack[val_start_idx..]) -} - /// A server-side HTTP request that may or may not be multipart. /// /// May be implemented by mutable references if providing the request or body by-value is @@ -424,18 +345,18 @@ pub struct MultipartField<'a, B: 'a> { impl<'a, B: Read + 'a> MultipartField<'a, B> { fn read_from(multipart: &'a mut Multipart) -> io::Result>> { - let cont_disp = match multipart.read_content_disposition() { - Ok(Some(cont_disp)) => cont_disp, + let field_headers = match multipart.read_field_headers() { + Ok(Some(headers)) => headers, Ok(None) => return Ok(None), - Err(err) => return Err(err), - }; + Err(err) => return Err(err) + }; - let data = match try!(multipart.read_content_type()) { + let data = match field_headers.cont_type { Some(content_type) => { let _ = try!(multipart.read_line()); // Consume empty line MultipartData::File( MultipartFile::from_stream( - cont_disp.filename, + field_headers.cont_disp.filename, content_type.val, &mut multipart.source, ) @@ -450,7 +371,7 @@ impl<'a, B: Read + 'a> MultipartField<'a, B> { Ok(Some( MultipartField { - name: cont_disp.field_name, + name: field_headers.cont_disp.field_name, data: data, } )) diff --git a/multipart/src/server/parse.rs b/multipart/src/server/parse.rs new file mode 100644 index 000000000..94bf04321 --- /dev/null +++ b/multipart/src/server/parse.rs @@ -0,0 +1,220 @@ +// Copyright 2016 `multipart` Crate Developers +// +// Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be +// copied, modified, or distributed except according to those terms. + +extern crate httparse; + +use self::httparse::{EMPTY_HEADER, Header, Status}; + +use super::boundary::BoundaryReader; + +use mime::Mime; + +use std::io::{self, BufRead}; +use std::str; + +macro_rules! try_io( + ($try:expr) => ( + { + use std::io::{Error, ErrorKind}; + match $try { + Ok(val) => val, + Err(e) => return Err(Error::new(ErrorKind::InvalidData, format!("{:?}", e))), + } + } + ) +); + +macro_rules! assert_log_ret_none ( + ($expr, $else_:expr) => ( + if !$expr { + $else_; + return None; + } + ) +); + +const EMPTY_STR_HEADER: StrHeader<'static> = StrHeader { + name: "", + val: "", +}; + +#[derive(Copy, Clone, Debug)] +struct StrHeader<'a> { + name: &'a str, + val: &'a str, +} + +pub struct MultipartHeaders { + pub cont_disp: ContentDisp, + pub cont_type: Option, +} + +impl MultipartHeaders { + pub fn parse(r: &mut R) -> io::Result> { + const HEADER_LEN: usize = 4; + + let mut consume = 0; + + let mut headers = [EMPTY_STR_HEADER; HEADER_LEN]; + let mut header_len = 0; + + { + let mut raw_headers = [EMPTY_HEADER; HEADER_LEN]; + + loop { + let mut buf = try!(r.fill_buf()); + + match try_io!(httparse::parse_headers(buf, &mut raw_headers)) { + Status::Complete((consume_, raw_headers)) => { + consume = consume_; + header_len = raw_headers.len(); + break; + }, + Status::Partial => (), + } + } + + debug!("Parsed field headers: {:?}", raw_headers); + + for (raw, header) in raw_headers.iter().take(header_len).zip(&mut headers) { + header.name = raw.name; + header.val = try!(io_str_utf8(raw.value)); + } + } + + let headers = &headers[..header_len]; + + r.consume(consume); + + Ok(Self::read_from(headers)) + } + + fn read_from(headers: &[StrHeader]) -> Option { + let cont_disp = try_opt!( + ContentDisp::read_from(headers), + debug!("Failed to read Content-Disposition") + ); + + let cont_type = ContentType::read_from(headers); + + Some(MultipartHeaders { + cont_disp: cont_disp, + cont_type: cont_type, + }) + } +} + +pub struct ContentDisp { + pub field_name: String, + pub filename: Option, +} + +impl ContentDisp { + fn read_from(headers: &[StrHeader]) -> Option { + if headers.is_empty() { + return None; + } + + const CONT_DISP: &'static str = "Content-Disposition"; + + let header = try_opt!( + find_header(headers, CONT_DISP), + error!("Field headers did not contain Content-Disposition header (required)") + ); + + const NAME: &'static str = "name=\""; + const FILENAME: &'static str = "filename=\""; + + let after_disp_type = { + let (disp_type, after_disp_type) = try_opt!( + split_once(header.val, ';'), + error!("Expected additional data after Content-Disposition type, got {:?}", + header.val) + ); + + + if disp_type.trim() != "form-data" { + error!("Unexpected Content-Disposition value: {:?}", disp_type); + return None; + }; + + after_disp_type + }; + + let (field_name, after_field_name) = try_opt!( + get_str_after(NAME, '"', after_disp_type), + error!("Expected field name and maybe filename, got {:?}", after_disp_type) + ); + + let filename = get_str_after(FILENAME, '"', after_field_name) + .map(|(filename, _)| filename.to_owned()); + + Some(ContentDisp { field_name: field_name.to_owned(), filename: filename }) + } +} + +pub struct ContentType { + pub val: Mime, + #[allow(dead_code)] + pub boundary: Option, +} + +impl ContentType { + fn read_from(headers: &[StrHeader]) -> Option { + const CONTENT_TYPE: &'static str = "Content-Type"; + + let header = try_opt!(find_header(headers, CONTENT_TYPE)); + + const BOUNDARY: &'static str = "boundary=\""; + + if let Some((cont_type, after_cont_type)) = get_str_after(CONTENT_TYPE, ';', header.val) { + let content_type = read_content_type(cont_type.trim()); + + let boundary = get_str_after(BOUNDARY, '"', after_cont_type).map(|tup| tup.0.into()); + + Some(ContentType { + val: content_type, + boundary: boundary, + }) + } else { + get_remainder_after(CONTENT_TYPE, header.val).map(|cont_type| { + let content_type = read_content_type(cont_type.trim()); + ContentType { val: content_type, boundary: None } + }) + } + } +} + +fn read_content_type(cont_type: &str) -> Mime { + cont_type.parse().ok().unwrap_or_else(::mime_guess::octet_stream) +} + + +fn split_once(s: &str, delim: char) -> Option<(&str, &str)> { + s.find(delim).map(|idx| s.split_at(idx)) +} + +/// Get the string after `needle` in `haystack`, stopping before `end_val_delim` +fn get_str_after<'a>(needle: &str, end_val_delim: char, haystack: &'a str) -> Option<(&'a str, &'a str)> { + let val_start_idx = try_opt!(haystack.find(needle)) + needle.len(); + let val_end_idx = try_opt!(haystack[val_start_idx..].find(end_val_delim)) + val_start_idx; + Some((&haystack[val_start_idx..val_end_idx], &haystack[val_end_idx..])) +} + +/// Get everything after `needle` in `haystack` +fn get_remainder_after<'a>(needle: &str, haystack: &'a str) -> Option<(&'a str)> { + let val_start_idx = try_opt!(haystack.find(needle)) + needle.len(); + Some(&haystack[val_start_idx..]) +} + +fn io_str_utf8(buf: &[u8]) -> io::Result<&str> { + str::from_utf8(buf).map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e)) +} + +fn find_header<'a, 'b>(headers: &'a [StrHeader<'b>], name: &str) -> Option<&'a StrHeader<'b>> { + headers.iter().find(|header| header.name == name) +} \ No newline at end of file From a72a987a5d1199973cb51e404833a81f453cd032 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Mon, 17 Oct 2016 21:36:01 -0700 Subject: [PATCH 187/453] Complete `httparse` integration --- multipart/src/server/mod.rs | 10 --------- multipart/src/server/parse.rs | 42 +++++++++++++++++------------------ 2 files changed, 21 insertions(+), 31 deletions(-) diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index 31cb79ff2..6c7733ce0 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -227,15 +227,6 @@ impl Multipart { Ok(()) } - fn read_line(&mut self) -> io::Result<&str> { - self.line_buf.clear(); - - match self.source.read_line(&mut self.line_buf) { - Ok(read) => Ok(&self.line_buf[..read]), - Err(err) => Err(err), - } - } - fn read_to_string(&mut self) -> io::Result<&str> { self.line_buf.clear(); @@ -353,7 +344,6 @@ impl<'a, B: Read + 'a> MultipartField<'a, B> { let data = match field_headers.cont_type { Some(content_type) => { - let _ = try!(multipart.read_line()); // Consume empty line MultipartData::File( MultipartFile::from_stream( field_headers.cont_disp.filename, diff --git a/multipart/src/server/parse.rs b/multipart/src/server/parse.rs index 94bf04321..e217b4bc3 100644 --- a/multipart/src/server/parse.rs +++ b/multipart/src/server/parse.rs @@ -7,9 +7,7 @@ extern crate httparse; -use self::httparse::{EMPTY_HEADER, Header, Status}; - -use super::boundary::BoundaryReader; +use self::httparse::{EMPTY_HEADER, Status}; use mime::Mime; @@ -57,16 +55,17 @@ impl MultipartHeaders { pub fn parse(r: &mut R) -> io::Result> { const HEADER_LEN: usize = 4; - let mut consume = 0; + // These are only written once so they don't need to be `mut` or initialized. + let consume; + let header_len; let mut headers = [EMPTY_STR_HEADER; HEADER_LEN]; - let mut header_len = 0; { let mut raw_headers = [EMPTY_HEADER; HEADER_LEN]; loop { - let mut buf = try!(r.fill_buf()); + let buf = try!(r.fill_buf()); match try_io!(httparse::parse_headers(buf, &mut raw_headers)) { Status::Complete((consume_, raw_headers)) => { @@ -78,8 +77,6 @@ impl MultipartHeaders { } } - debug!("Parsed field headers: {:?}", raw_headers); - for (raw, header) in raw_headers.iter().take(header_len).zip(&mut headers) { header.name = raw.name; header.val = try!(io_str_utf8(raw.value)); @@ -88,6 +85,8 @@ impl MultipartHeaders { let headers = &headers[..header_len]; + debug!("Parsed field headers: {:?}", headers); + r.consume(consume); Ok(Self::read_from(headers)) @@ -167,24 +166,31 @@ impl ContentType { fn read_from(headers: &[StrHeader]) -> Option { const CONTENT_TYPE: &'static str = "Content-Type"; - let header = try_opt!(find_header(headers, CONTENT_TYPE)); + let header = try_opt!( + find_header(headers, CONTENT_TYPE), + debug!("Content-Type header not found for field.") + ); const BOUNDARY: &'static str = "boundary=\""; - if let Some((cont_type, after_cont_type)) = get_str_after(CONTENT_TYPE, ';', header.val) { + if let Some((cont_type, after_cont_type)) = split_once(header.val, ';') { + debug!("Found Content-Type: {:?}", cont_type); + let content_type = read_content_type(cont_type.trim()); - let boundary = get_str_after(BOUNDARY, '"', after_cont_type).map(|tup| tup.0.into()); + let boundary = get_str_after(BOUNDARY, '"', after_cont_type) + .map(|tup| tup.0.to_string()); + + debug!("Found sub-boundary: {:?}", boundary); Some(ContentType { val: content_type, boundary: boundary, }) } else { - get_remainder_after(CONTENT_TYPE, header.val).map(|cont_type| { - let content_type = read_content_type(cont_type.trim()); - ContentType { val: content_type, boundary: None } - }) + debug!("Found Content-Type: {:?}", header.val); + let content_type = read_content_type(header.val.trim()); + Some(ContentType { val: content_type, boundary: None }) } } } @@ -205,12 +211,6 @@ fn get_str_after<'a>(needle: &str, end_val_delim: char, haystack: &'a str) -> Op Some((&haystack[val_start_idx..val_end_idx], &haystack[val_end_idx..])) } -/// Get everything after `needle` in `haystack` -fn get_remainder_after<'a>(needle: &str, haystack: &'a str) -> Option<(&'a str)> { - let val_start_idx = try_opt!(haystack.find(needle)) + needle.len(); - Some(&haystack[val_start_idx..]) -} - fn io_str_utf8(buf: &[u8]) -> io::Result<&str> { str::from_utf8(buf).map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e)) } From 57c17f3589b29dfda9ad0768e992334ff1ae76be Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Mon, 17 Oct 2016 22:25:57 -0700 Subject: [PATCH 188/453] Remove mention of empty line (httparse eats it regardless) --- multipart/src/server/mod.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index 6c7733ce0..2beac7344 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -353,8 +353,7 @@ impl<'a, B: Read + 'a> MultipartField<'a, B> { ) }, None => { - // Empty line consumed by read_content_type() - let text = try!(multipart.read_to_string()); + let text = try!(multipart.read_to_string()); MultipartData::Text(&text) }, }; From 49875b80b0f30330a0a753cb7aaa09c73cf7d371 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Tue, 18 Oct 2016 18:11:18 -0700 Subject: [PATCH 189/453] Yeah, nevermind --- multipart/Cargo.toml | 19 ++---- multipart/src/lib.rs | 8 ++- multipart/src/server/boundary.rs | 8 +-- multipart/src/server/{parse.rs => field.rs} | 73 +++++++++++---------- multipart/src/server/mod.rs | 12 ++-- 5 files changed, 60 insertions(+), 60 deletions(-) rename multipart/src/server/{parse.rs => field.rs} (74%) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 164bc6e81..fb0a1f814 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -22,13 +22,10 @@ mime_guess = "1.8" rand = "0.3" tempdir = ">=0.3.4" -[dependencies.buf_redux] -optional = true -version = "0.3" - -[dependencies.httparse] -optional = true -version = "1.1" +#Server Dependencies +buf_redux = { version = "0.3", optional = true } +httparse = { version = "1.1", optional = true } +memchr = {version = "0.1", optional = true } [dependencies.hyper] optional = true @@ -39,10 +36,6 @@ version = "0.9" optional = true version = "0.4" -[dependencies.memchr] -optional = true -version = "0.1" - # NOTE: use `nickel_` feature, as `hyper` feature is required. [dependencies.nickel] optional = true @@ -58,6 +51,6 @@ env_logger = "0.3" [features] client = [] default = ["hyper", "server", "client"] -server = ["buf_redux", "memchr", "httparse"] +server = ["buf_redux", "httparse", "memchr"] nickel_ = ["nickel", "hyper"] -all = ["iron", "nickel_", "tiny_http"] +all = ["iron", "nickel_", "tiny_http"] \ No newline at end of file diff --git a/multipart/src/lib.rs b/multipart/src/lib.rs index 018ed9c47..6c4380acd 100644 --- a/multipart/src/lib.rs +++ b/multipart/src/lib.rs @@ -27,14 +27,16 @@ //! See the [`server::nickel`](server/nickel/index.html) module for more information. Enables the `hyper` //! feature. #![warn(missing_docs)] -#[macro_use] extern crate log; -#[cfg(test)] extern crate env_logger; +#[macro_use] +extern crate log; + +#[cfg(test)] +extern crate env_logger; extern crate mime; extern crate mime_guess; extern crate rand; - extern crate tempdir; #[cfg(feature = "hyper")] diff --git a/multipart/src/server/boundary.rs b/multipart/src/server/boundary.rs index 0f842fbe2..75dda5694 100644 --- a/multipart/src/server/boundary.rs +++ b/multipart/src/server/boundary.rs @@ -4,11 +4,11 @@ // http://apache.org/licenses/LICENSE-2.0> or the MIT license , at your option. This file may not be // copied, modified, or distributed except according to those terms. -extern crate buf_redux; -extern crate memchr; -use self::buf_redux::BufReader; -use self::memchr::memchr; +//! Boundary parsing for `multipart` requests. + +use super::buf_redux::BufReader; +use super::memchr::memchr; use std::cmp; use std::borrow::Borrow; diff --git a/multipart/src/server/parse.rs b/multipart/src/server/field.rs similarity index 74% rename from multipart/src/server/parse.rs rename to multipart/src/server/field.rs index e217b4bc3..5a9818a34 100644 --- a/multipart/src/server/parse.rs +++ b/multipart/src/server/field.rs @@ -5,11 +5,11 @@ // http://opensource.org/licenses/MIT>, at your option. This file may not be // copied, modified, or distributed except according to those terms. -extern crate httparse; +//! `multipart` field header parsing. -use self::httparse::{EMPTY_HEADER, Status}; +use super::httparse::{self, EMPTY_HEADER, Status}; -use mime::Mime; +use mime::{Attr, Mime, Value}; use std::io::{self, BufRead}; use std::str; @@ -46,13 +46,17 @@ struct StrHeader<'a> { val: &'a str, } -pub struct MultipartHeaders { +/// The headers that (may) appear before a `multipart/form-data` field. +pub struct FieldHeaders { + /// The `Content-Disposition` header, required. pub cont_disp: ContentDisp, + /// The `Content-Type` header, optional. pub cont_type: Option, } -impl MultipartHeaders { - pub fn parse(r: &mut R) -> io::Result> { +impl FieldHeaders { + /// Parse the field headers from the passed `BufRead`, consuming the relevant bytes. + pub fn parse(r: &mut R) -> io::Result> { const HEADER_LEN: usize = 4; // These are only written once so they don't need to be `mut` or initialized. @@ -92,7 +96,7 @@ impl MultipartHeaders { Ok(Self::read_from(headers)) } - fn read_from(headers: &[StrHeader]) -> Option { + fn read_from(headers: &[StrHeader]) -> Option { let cont_disp = try_opt!( ContentDisp::read_from(headers), debug!("Failed to read Content-Disposition") @@ -100,15 +104,18 @@ impl MultipartHeaders { let cont_type = ContentType::read_from(headers); - Some(MultipartHeaders { + Some(FieldHeaders { cont_disp: cont_disp, cont_type: cont_type, }) } } +/// The `Content-Disposition` header. pub struct ContentDisp { + /// The name of the `multipart/form-data` field. pub field_name: String, + /// The optional filename for this field. pub filename: Option, } @@ -145,21 +152,25 @@ impl ContentDisp { }; let (field_name, after_field_name) = try_opt!( - get_str_after(NAME, '"', after_disp_type), + get_str_after(NAME, ';', after_disp_type), error!("Expected field name and maybe filename, got {:?}", after_disp_type) ); - let filename = get_str_after(FILENAME, '"', after_field_name) - .map(|(filename, _)| filename.to_owned()); + let field_name = trim_quotes(field_name); + + let filename = get_str_after(FILENAME, ';', after_field_name) + .map(|(filename, _)| trim_quotes(filename).to_owned()); Some(ContentDisp { field_name: field_name.to_owned(), filename: filename }) } } +/// The `Content-Type` header. pub struct ContentType { + /// The MIME type of the `multipart` field. + /// + /// May contain a sub-boundary parameter. pub val: Mime, - #[allow(dead_code)] - pub boundary: Option, } impl ContentType { @@ -171,27 +182,15 @@ impl ContentType { debug!("Content-Type header not found for field.") ); - const BOUNDARY: &'static str = "boundary=\""; - - if let Some((cont_type, after_cont_type)) = split_once(header.val, ';') { - debug!("Found Content-Type: {:?}", cont_type); - - let content_type = read_content_type(cont_type.trim()); - - let boundary = get_str_after(BOUNDARY, '"', after_cont_type) - .map(|tup| tup.0.to_string()); - - debug!("Found sub-boundary: {:?}", boundary); + // Boundary parameter will be parsed into the `Mime` + debug!("Found Content-Type: {:?}", header.val); + let content_type = read_content_type(header.val.trim()); + Some(ContentType { val: content_type }) + } - Some(ContentType { - val: content_type, - boundary: boundary, - }) - } else { - debug!("Found Content-Type: {:?}", header.val); - let content_type = read_content_type(header.val.trim()); - Some(ContentType { val: content_type, boundary: None }) - } + /// Get the optional boundary parameter for this `Content-Type`. + pub fn boundary(&self) -> Option<&str> { + self.val.get_param(Attr::Boundary).map(Value::as_str) } } @@ -199,15 +198,19 @@ fn read_content_type(cont_type: &str) -> Mime { cont_type.parse().ok().unwrap_or_else(::mime_guess::octet_stream) } - fn split_once(s: &str, delim: char) -> Option<(&str, &str)> { s.find(delim).map(|idx| s.split_at(idx)) } +fn trim_quotes(s: &str) -> &str { + s.trim_matches('"') +} + /// Get the string after `needle` in `haystack`, stopping before `end_val_delim` fn get_str_after<'a>(needle: &str, end_val_delim: char, haystack: &'a str) -> Option<(&'a str, &'a str)> { let val_start_idx = try_opt!(haystack.find(needle)) + needle.len(); - let val_end_idx = try_opt!(haystack[val_start_idx..].find(end_val_delim)) + val_start_idx; + let val_end_idx = haystack[val_start_idx..].find(end_val_delim) + .map_or(haystack.len(), |end_idx| end_idx + val_start_idx); Some((&haystack[val_start_idx..val_end_idx], &haystack[val_end_idx..])) } diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index 2beac7344..29c7f9f06 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -12,6 +12,9 @@ //! //! See the `Multipart` struct for more info. +extern crate buf_redux; +extern crate httparse; +extern crate memchr; use mime::Mime; @@ -25,7 +28,7 @@ use std::path::{Path, PathBuf}; use std::{io, mem, ptr}; use self::boundary::BoundaryReader; -use self::parse::MultipartHeaders; +use self::field::FieldHeaders; macro_rules! try_opt ( ($expr:expr) => ( @@ -46,8 +49,7 @@ macro_rules! try_opt ( ); mod boundary; - -mod parse; +mod field; #[cfg(feature = "hyper")] pub mod hyper; @@ -114,8 +116,8 @@ impl Multipart { MultipartField::read_from(self) } - fn read_field_headers(&mut self) -> io::Result> { - MultipartHeaders::parse(&mut self.source) + fn read_field_headers(&mut self) -> io::Result> { + FieldHeaders::parse(&mut self.source) } /// Call `f` for each entry in the multipart request. From 201103d445344fbb89ca45c4eea14df8455fa381 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Wed, 19 Oct 2016 01:00:14 -0700 Subject: [PATCH 190/453] Add a random number of hyphens to field and filenames --- multipart/src/local_test.rs | 40 ++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/multipart/src/local_test.rs b/multipart/src/local_test.rs index 56636bf28..e706b34e4 100644 --- a/multipart/src/local_test.rs +++ b/multipart/src/local_test.rs @@ -9,12 +9,11 @@ use client::HttpStream as ClientStream; use server::HttpRequest as ServerRequest; -use rand::Rng; -use rand::distributions::{Range, Sample}; +use rand::{self, Rng}; use std::collections::HashMap; -use std::io; use std::io::prelude::*; +use std::{io, iter}; #[derive(Debug)] struct TestFields { @@ -42,8 +41,10 @@ fn gen_test_fields() -> TestFields { const MIN_FIELDS: usize = 1; const MAX_FIELDS: usize = 3; - let texts_count = gen_range(MIN_FIELDS, MAX_FIELDS); - let files_count = gen_range(MIN_FIELDS, MAX_FIELDS); + let mut rng = rand::weak_rng(); + + let texts_count = rng.gen_range(MIN_FIELDS, MAX_FIELDS); + let files_count = rng.gen_range(MIN_FIELDS, MAX_FIELDS); TestFields { texts: (0..texts_count).map(|_| (gen_string(), gen_string())).collect(), @@ -51,29 +52,26 @@ fn gen_test_fields() -> TestFields { } } -fn gen_range(min: usize, max: usize) -> usize { - Range::new(min, max).sample(&mut ::rand::weak_rng()) -} - fn gen_string() -> String { const MIN_LEN: usize = 3; const MAX_LEN: usize = 8; + const MAX_DASHES: usize = 3; - let mut rng = ::rand::weak_rng(); - let str_len = gen_range(MIN_LEN, MAX_LEN); + let mut rng_1 = rand::weak_rng(); + let mut rng_2 = rand::weak_rng(); - rng.gen_ascii_chars().take(str_len).collect() + let str_len_1 = rng_1.gen_range(MIN_LEN, MAX_LEN + 1); + let str_len_2 = rng_2.gen_range(MIN_LEN, MAX_LEN + 1); + let num_dashes = rng_1.gen_range(0, MAX_DASHES + 1); + + rng_1.gen_ascii_chars().take(str_len_1) + .chain(iter::repeat('-').take(num_dashes)) + .chain(rng_2.gen_ascii_chars().take(str_len_2)) + .collect() } fn gen_bytes() -> Vec { - const MIN_LEN: usize = 8; - const MAX_LEN: usize = 16; - - let mut rng = ::rand::weak_rng(); - let bytes_len = gen_range(MIN_LEN, MAX_LEN); - - rng.gen_ascii_chars().take(bytes_len) - .map(|c| c as u8).collect() + gen_string().into_bytes() } fn test_client(test_fields: &TestFields) -> HttpBuffer { @@ -88,7 +86,7 @@ fn test_client(test_fields: &TestFields) -> HttpBuffer { // Intersperse file fields amongst text fields for (name, text) in &test_fields.texts { if let Some((file_name, file)) = test_files.next() { - multipart.write_stream(file_name, &mut &**file, None, None).unwrap(); + multipart.write_stream(file_name, &mut &**file, Some(file_name), None).unwrap(); } multipart.write_text(name, text).unwrap(); From 49ca18e08134ecdca669d4dca124d08949a04153 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Wed, 19 Oct 2016 01:00:35 -0700 Subject: [PATCH 191/453] Fix BoundaryReader looping infinitely on a benign dash --- multipart/src/server/boundary.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/src/server/boundary.rs b/multipart/src/server/boundary.rs index 75dda5694..20ee5dc29 100644 --- a/multipart/src/server/boundary.rs +++ b/multipart/src/server/boundary.rs @@ -68,7 +68,7 @@ impl BoundaryReader where R: Read { let test = &buf[self.search_idx .. self.search_idx + self.boundary.len()]; match first_nonmatching_idx(test, &self.boundary) { - Some(idx) => self.search_idx += idx, + Some(idx) => self.search_idx += idx + 1, None => self.boundary_read = true, } } else { From 881d974e9d0430ae9c23264097454fc5cb9df979 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Fri, 21 Oct 2016 22:29:42 -0700 Subject: [PATCH 192/453] Use new httparse with error::Error impl --- multipart/Cargo.toml | 26 ++++++++++++-------------- multipart/src/server/field.rs | 2 +- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index fb0a1f814..d818240a2 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -24,26 +24,24 @@ tempdir = ">=0.3.4" #Server Dependencies buf_redux = { version = "0.3", optional = true } -httparse = { version = "1.1", optional = true } memchr = {version = "0.1", optional = true } -[dependencies.hyper] -optional = true -default-features = false -version = "0.9" - -[dependencies.iron] -optional = true -version = "0.4" +# Optional Integrations +hyper = { version = "0.9", optional = true } +iron = { version = "0.4", optional = true } # NOTE: use `nickel_` feature, as `hyper` feature is required. -[dependencies.nickel] -optional = true -version = "0.9" +nickel = { optional = true, version = "0.9" } + +tiny_http = { version = "0.5", optional = true } -[dependencies.tiny_http] +[dependencies.httparse] +# FIXME: when updated version is released +#version = "1.1", +git = "https://github.com/seanmonstar/httparse" optional = true -version = "0.5" +default-features = false + [dev-dependencies] env_logger = "0.3" diff --git a/multipart/src/server/field.rs b/multipart/src/server/field.rs index 5a9818a34..418ccd77b 100644 --- a/multipart/src/server/field.rs +++ b/multipart/src/server/field.rs @@ -20,7 +20,7 @@ macro_rules! try_io( use std::io::{Error, ErrorKind}; match $try { Ok(val) => val, - Err(e) => return Err(Error::new(ErrorKind::InvalidData, format!("{:?}", e))), + Err(e) => return Err(Error::new(ErrorKind::InvalidData, e)), } } ) From a08b5515981b9064c95cdb05005be4a24a6e1013 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Thu, 10 Nov 2016 18:16:13 -0800 Subject: [PATCH 193/453] Tentative fix for boundary reading problems --- multipart/.gitignore | 1 + multipart/Cargo.toml | 8 ++--- multipart/src/bin/test_multipart.rs | 50 +++++++++++++++++++++++++++++ multipart/src/server/boundary.rs | 11 +++++-- multipart/src/server/field.rs | 4 +-- multipart/src/server/mod.rs | 2 ++ 6 files changed, 65 insertions(+), 11 deletions(-) create mode 100644 multipart/src/bin/test_multipart.rs diff --git a/multipart/.gitignore b/multipart/.gitignore index f55131aa6..fc01f5899 100644 --- a/multipart/.gitignore +++ b/multipart/.gitignore @@ -16,3 +16,4 @@ .idea/ +dump.bin \ No newline at end of file diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index d818240a2..43d58a24d 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -24,7 +24,7 @@ tempdir = ">=0.3.4" #Server Dependencies buf_redux = { version = "0.3", optional = true } -memchr = {version = "0.1", optional = true } +memchr = { version = "0.1", optional = true } # Optional Integrations hyper = { version = "0.9", optional = true } @@ -36,12 +36,8 @@ nickel = { optional = true, version = "0.9" } tiny_http = { version = "0.5", optional = true } [dependencies.httparse] -# FIXME: when updated version is released -#version = "1.1", -git = "https://github.com/seanmonstar/httparse" +version = "1.2" optional = true -default-features = false - [dev-dependencies] env_logger = "0.3" diff --git a/multipart/src/bin/test_multipart.rs b/multipart/src/bin/test_multipart.rs new file mode 100644 index 000000000..c9a85476b --- /dev/null +++ b/multipart/src/bin/test_multipart.rs @@ -0,0 +1,50 @@ +extern crate multipart; + +extern crate log; + +use log::{LogRecord, LogMetadata, LogLevelFilter}; + +use multipart::server::Multipart; + +use std::fs::File; +use std::env; + +const LOG_LEVEL: LogLevelFilter = LogLevelFilter::Off; + +struct SimpleLogger; + +impl log::Log for SimpleLogger { + fn enabled(&self, metadata: &LogMetadata) -> bool { + LOG_LEVEL.to_log_level() + .map_or(false, |level| metadata.level() <= level) + } + + fn log(&self, record: &LogRecord) { + if self.enabled(record.metadata()) { + println!("{} - {}", record.level(), record.args()); + } + } +} + +fn main() { + log::set_logger(|max_log_level| { + max_log_level.set(LOG_LEVEL); + Box::new(SimpleLogger) + }); + + let mut args = env::args().skip(1); + + let boundary = args.next().expect("Boundary must be provided as the first argument"); + + let file = args.next().expect("Filename must be provided as the second argument"); + + let file = File::open(file).expect("Could not open file"); + + let mut multipart = Multipart::with_body(file, boundary); + + while let Some(field) = multipart.read_entry().unwrap() { + println!("Read field: {:?}", field.name); + } + + println!("All entries read!"); +} \ No newline at end of file diff --git a/multipart/src/server/boundary.rs b/multipart/src/server/boundary.rs index 20ee5dc29..005fe5103 100644 --- a/multipart/src/server/boundary.rs +++ b/multipart/src/server/boundary.rs @@ -57,8 +57,6 @@ impl BoundaryReader where R: Read { let maybe_boundary = memchr(self.boundary[0], lookahead); - debug!("maybe_boundary: {:?}", maybe_boundary); - self.search_idx = match maybe_boundary { Some(boundary_start) => self.search_idx + boundary_start, None => buf.len(), @@ -67,8 +65,15 @@ impl BoundaryReader where R: Read { if self.search_idx + self.boundary.len() <= buf.len() { let test = &buf[self.search_idx .. self.search_idx + self.boundary.len()]; + if log_enabled!(LogLevel::Trace) { + trace!("Possible boundary: {:?}", String::from_utf8_lossy(test)); + } + match first_nonmatching_idx(test, &self.boundary) { - Some(idx) => self.search_idx += idx + 1, + Some(idx) => { + debug!("First nonmatching idx: {}", idx); + self.search_idx += idx + 1; + }, None => self.boundary_read = true, } } else { diff --git a/multipart/src/server/field.rs b/multipart/src/server/field.rs index 418ccd77b..bbfdd000c 100644 --- a/multipart/src/server/field.rs +++ b/multipart/src/server/field.rs @@ -132,8 +132,8 @@ impl ContentDisp { error!("Field headers did not contain Content-Disposition header (required)") ); - const NAME: &'static str = "name=\""; - const FILENAME: &'static str = "filename=\""; + const NAME: &'static str = "name="; + const FILENAME: &'static str = "filename="; let after_disp_type = { let (disp_type, after_disp_type) = try_opt!( diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index 29c7f9f06..e6fa9b00a 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -239,6 +239,8 @@ impl Multipart { } fn consume_boundary(&mut self) -> io::Result { + debug!("Consume boundary!"); + try!(self.source.consume_boundary()); let mut out = [0; 2]; From 5f88352727558671f59b9529705aad5c4c4cd486 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Thu, 10 Nov 2016 18:18:33 -0800 Subject: [PATCH 194/453] Set `debug` log level in `test_multipart.rs` --- multipart/src/bin/test_multipart.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/src/bin/test_multipart.rs b/multipart/src/bin/test_multipart.rs index c9a85476b..114e4326f 100644 --- a/multipart/src/bin/test_multipart.rs +++ b/multipart/src/bin/test_multipart.rs @@ -9,7 +9,7 @@ use multipart::server::Multipart; use std::fs::File; use std::env; -const LOG_LEVEL: LogLevelFilter = LogLevelFilter::Off; +const LOG_LEVEL: LogLevelFilter = LogLevelFilter::Debug; struct SimpleLogger; From c67766e9620deb22a340427b463f6691b569fcd6 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Thu, 10 Nov 2016 18:53:10 -0800 Subject: [PATCH 195/453] Add an accessor for the boundary of `PreparedFields` --- multipart/src/client/lazy.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/multipart/src/client/lazy.rs b/multipart/src/client/lazy.rs index f964f49d3..722446dc9 100644 --- a/multipart/src/client/lazy.rs +++ b/multipart/src/client/lazy.rs @@ -341,8 +341,8 @@ impl<'d> PreparedFields<'d> { self.content_len } - #[cfg_attr(not(feature = "hyper"), allow(dead_code))] - fn boundary(&self) -> &str { + /// Get the boundary that was used to serialize the request. + pub fn boundary(&self) -> &str { &self.boundary } } From fdf75a6f5ba7428abfef4f6b3883a435b2427770 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Thu, 10 Nov 2016 18:53:49 -0800 Subject: [PATCH 196/453] Bump version --- multipart/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 43d58a24d..dd6a0e086 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "multipart" -version = "0.9.0-alpha.1" +version = "0.9.0-alpha.2" authors = ["Austin Bonander "] From aeef26d90a93394369f00a412d0b81d2f2320252 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Mon, 21 Nov 2016 04:12:14 -0800 Subject: [PATCH 197/453] Upgrade buf_redux --- multipart/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index dd6a0e086..b19acf9c4 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -23,7 +23,7 @@ rand = "0.3" tempdir = ">=0.3.4" #Server Dependencies -buf_redux = { version = "0.3", optional = true } +buf_redux = { version = "0.5", optional = true } memchr = { version = "0.1", optional = true } # Optional Integrations From f07d7295739de38ae671cf783fe9a823a29c8c2f Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Mon, 21 Nov 2016 04:15:05 -0800 Subject: [PATCH 198/453] Fix logic errors in boundary reading --- multipart/src/server/boundary.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/multipart/src/server/boundary.rs b/multipart/src/server/boundary.rs index 005fe5103..49e50281f 100644 --- a/multipart/src/server/boundary.rs +++ b/multipart/src/server/boundary.rs @@ -41,7 +41,10 @@ impl BoundaryReader where R: Read { fn read_to_boundary(&mut self) -> io::Result<&[u8]> { use log::LogLevel; - let buf = try!(fill_buf_min(&mut self.buf, self.boundary.len() * 2)); + // Make sure there's enough bytes in the buffer to positively identify the boundary. + let min_len = self.search_idx + (self.boundary.len() * 2); + + let buf = try!(fill_buf_min(&mut self.buf, min_len)); if log_enabled!(LogLevel::Trace) { trace!("Buf: {:?}", String::from_utf8_lossy(buf)); @@ -72,7 +75,7 @@ impl BoundaryReader where R: Read { match first_nonmatching_idx(test, &self.boundary) { Some(idx) => { debug!("First nonmatching idx: {}", idx); - self.search_idx += idx + 1; + self.search_idx += idx; }, None => self.boundary_read = true, } @@ -86,7 +89,6 @@ impl BoundaryReader where R: Read { buf.len(), self.search_idx, self.boundary_read ); - let mut buf_end = self.search_idx; if self.boundary_read && self.search_idx >= 2 { From fb04fef5bcb114d891641c3d67e2602ee9eb8eab Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Mon, 21 Nov 2016 04:27:36 -0800 Subject: [PATCH 199/453] Get rid of unused_result warning --- multipart/src/bin/test_multipart.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/src/bin/test_multipart.rs b/multipart/src/bin/test_multipart.rs index 114e4326f..79a353fb1 100644 --- a/multipart/src/bin/test_multipart.rs +++ b/multipart/src/bin/test_multipart.rs @@ -30,7 +30,7 @@ fn main() { log::set_logger(|max_log_level| { max_log_level.set(LOG_LEVEL); Box::new(SimpleLogger) - }); + }).expect("Could not initialize logger"); let mut args = env::args().skip(1); From b043c0c6f6971f489a99fff2822dcbccee93a45e Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Mon, 21 Nov 2016 04:28:41 -0800 Subject: [PATCH 200/453] Move all types concerning individual fields to src/server/field.rs, add safe version of `prepend_str` --- multipart/Cargo.toml | 1 + multipart/src/lib.rs | 1 + multipart/src/server/field.rs | 265 ++++++++++++++++++++++++++++++++- multipart/src/server/mod.rs | 266 ++-------------------------------- 4 files changed, 275 insertions(+), 258 deletions(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index b19acf9c4..34177c552 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -47,4 +47,5 @@ client = [] default = ["hyper", "server", "client"] server = ["buf_redux", "httparse", "memchr"] nickel_ = ["nickel", "hyper"] +nightly = [] all = ["iron", "nickel_", "tiny_http"] \ No newline at end of file diff --git a/multipart/src/lib.rs b/multipart/src/lib.rs index 6c4380acd..efdffea56 100644 --- a/multipart/src/lib.rs +++ b/multipart/src/lib.rs @@ -27,6 +27,7 @@ //! See the [`server::nickel`](server/nickel/index.html) module for more information. Enables the `hyper` //! feature. #![warn(missing_docs)] +#![cfg_attr(feature = "nightly", feature(insert_str))] #[macro_use] extern crate log; diff --git a/multipart/src/server/field.rs b/multipart/src/server/field.rs index bbfdd000c..2bed6fc9c 100644 --- a/multipart/src/server/field.rs +++ b/multipart/src/server/field.rs @@ -9,11 +9,19 @@ use super::httparse::{self, EMPTY_HEADER, Status}; +use super::Multipart; + +use super::boundary::BoundaryReader; + use mime::{Attr, Mime, Value}; -use std::io::{self, BufRead}; +use std::io::{self, Read, BufRead, Write}; +use std::fs::{self, File}; +use std::path::{Path, PathBuf}; use std::str; +const RANDOM_FILENAME_LEN: usize = 12; + macro_rules! try_io( ($try:expr) => ( { @@ -189,11 +197,244 @@ impl ContentType { } /// Get the optional boundary parameter for this `Content-Type`. + #[allow(dead_code)] pub fn boundary(&self) -> Option<&str> { self.val.get_param(Attr::Boundary).map(Value::as_str) } } +/// A field in a multipart request. May be either text or a binary stream (file). +#[derive(Debug)] +pub struct MultipartField<'a, B: 'a> { + /// The field's name from the form + pub name: String, + /// The data of the field. Can be text or binary. + pub data: MultipartData<'a, B>, +} + +pub fn read_field(multipart: &mut Multipart) -> io::Result>> { + let field_headers = match multipart.read_field_headers() { + Ok(Some(headers)) => headers, + Ok(None) => return Ok(None), + Err(err) => return Err(err) + }; + + let data = match field_headers.cont_type { + Some(content_type) => { + MultipartData::File( + MultipartFile::from_stream( + field_headers.cont_disp.filename, + content_type.val, + &mut multipart.source, + ) + ) + }, + None => { + let text = try!(multipart.read_to_string()); + MultipartData::Text(&text) + }, + }; + + Ok(Some( + MultipartField { + name: field_headers.cont_disp.field_name, + data: data, + } + )) +} + +/// The data of a field in a `multipart/form-data` request. +#[derive(Debug)] +pub enum MultipartData<'a, B: 'a> { + /// The field's payload is a text string. + Text(&'a str), + /// The field's payload is a binary stream (file). + File(MultipartFile<'a, B>), + // TODO: Support multiple files per field (nested boundaries) + // MultiFiles(Vec), +} + +impl<'a, B> MultipartData<'a, B> { + /// Borrow this payload as a text field, if possible. + pub fn as_text(&self) -> Option<&str> { + match *self { + MultipartData::Text(ref s) => Some(s), + _ => None, + } + } + + /// Borrow this payload as a file field, if possible. + /// Mutably borrows so the contents can be read. + pub fn as_file(&mut self) -> Option<&mut MultipartFile<'a, B>> { + match *self { + MultipartData::File(ref mut file) => Some(file), + _ => None, + } + } +} + +/// A representation of a file in HTTP `multipart/form-data`. +/// +/// Note that the file is not yet saved to the local filesystem; +/// instead, this struct exposes `Read` and `BufRead` impls which point +/// to the beginning of the file's contents in the HTTP stream. +/// +/// You can read it to EOF, or use one of the `save_*()` methods here +/// to save it to disk. +#[derive(Debug)] +pub struct MultipartFile<'a, B: 'a> { + filename: Option, + content_type: Mime, + stream: &'a mut BoundaryReader, +} + +impl<'a, B: Read> MultipartFile<'a, B> { + fn from_stream(filename: Option, + content_type: Mime, + stream: &'a mut BoundaryReader) -> MultipartFile<'a, B> { + MultipartFile { + filename: filename, + content_type: content_type, + stream: stream, + } + } + + /// Save this file to the given output stream. + /// + /// If successful, returns the number of bytes written. + /// + /// Retries when `io::Error::kind() == io::ErrorKind::Interrupted`. + pub fn save_to(&mut self, mut out: W) -> io::Result { + retry_on_interrupt(|| io::copy(self.stream, &mut out)) + } + + /// Save this file to the given output stream, **truncated** to `limit` + /// (no more than `limit` bytes will be written out). + /// + /// If successful, returns the number of bytes written. + /// + /// Retries when `io::Error::kind() == io::ErrorKind::Interrupted`. + pub fn save_to_limited(&mut self, mut out: W, limit: u64) -> io::Result { + retry_on_interrupt(|| io::copy(&mut self.stream.take(limit), &mut out)) + } + + /// Save this file to `path`. + /// + /// Returns the saved file info on success, or any errors otherwise. + /// + /// Retries when `io::Error::kind() == io::ErrorKind::Interrupted`. + pub fn save_as>(&mut self, path: P) -> io::Result { + let path = path.into(); + let file = try!(create_full_path(&path)); + let size = try!(self.save_to(file)); + + Ok(SavedFile { + path: path, + filename: self.filename.clone(), + size: size, + }) + } + + /// Save this file in the directory pointed at by `dir`, + /// using a random alphanumeric string as the filename. + /// + /// Any missing directories in the `dir` path will be created. + /// + /// Returns the saved file's info on success, or any errors otherwise. + /// + /// Retries when `io::Error::kind() == io::ErrorKind::Interrupted`. + pub fn save_in>(&mut self, dir: P) -> io::Result { + let path = dir.as_ref().join(::random_alphanumeric(RANDOM_FILENAME_LEN)); + self.save_as(path) + } + + /// Save this file to `path`, **truncated** to `limit` (no more than `limit` bytes will be written out). + /// + /// Any missing directories in the `dir` path will be created. + /// + /// Returns the saved file's info on success, or any errors otherwise. + /// + /// Retries when `io::Error::kind() == io::ErrorKind::Interrupted`. + pub fn save_as_limited>(&mut self, path: P, limit: u64) -> io::Result { + let path = path.into(); + let file = try!(create_full_path(&path)); + let size = try!(self.save_to_limited(file, limit)); + + Ok(SavedFile { + path: path, + filename: self.filename.clone(), + size: size, + }) + } + + /// Save this file in the directory pointed at by `dir`, + /// using a random alphanumeric string as the filename. + /// + /// **Truncates** file to `limit` (no more than `limit` bytes will be written out). + /// + /// Any missing directories in the `dir` path will be created. + /// + /// Returns the saved file's info on success, or any errors otherwise. + /// + /// Retries when `io::Error::kind() == io::ErrorKind::Interrupted`. + pub fn save_in_limited>(&mut self, dir: P, limit: u64) -> io::Result { + let path = dir.as_ref().join(::random_alphanumeric(RANDOM_FILENAME_LEN)); + self.save_as_limited(path, limit) + } + + /// Get the filename of this entry, if supplied. + /// + /// ##Warning + /// You should treat this value as untrustworthy because it is an arbitrary string provided by + /// the client. You should *not* blindly append it to a directory path and save the file there, + /// as such behavior could easily be exploited by a malicious client. + pub fn filename(&self) -> Option<&str> { + self.filename.as_ref().map(String::as_ref) + } + + /// Get the MIME type (`Content-Type` value) of this file, if supplied by the client, + /// or `"applicaton/octet-stream"` otherwise. + pub fn content_type(&self) -> &Mime { + &self.content_type + } +} + +impl<'a, B: Read> Read for MultipartFile<'a, B> { + fn read(&mut self, buf: &mut [u8]) -> io::Result{ + self.stream.read(buf) + } +} + +impl<'a, B: Read> BufRead for MultipartFile<'a, B> { + fn fill_buf(&mut self) -> io::Result<&[u8]> { + self.stream.fill_buf() + } + + fn consume(&mut self, amt: usize) { + self.stream.consume(amt) + } +} + +/// A file saved to the local filesystem from a multipart request. +#[derive(Debug)] +pub struct SavedFile { + /// The complete path this file was saved at. + pub path: PathBuf, + + /// The original filename of this file, if one was provided in the request. + /// + /// ##Warning + /// You should treat this value as untrustworthy because it is an arbitrary string provided by + /// the client. You should *not* blindly append it to a directory path and save the file there, + /// as such behavior could easily be exploited by a malicious client. + pub filename: Option, + + /// The number of bytes written to the disk; may be truncated. + pub size: u64, +} + + + fn read_content_type(cont_type: &str) -> Mime { cont_type.parse().ok().unwrap_or_else(::mime_guess::octet_stream) } @@ -220,4 +461,26 @@ fn io_str_utf8(buf: &[u8]) -> io::Result<&str> { fn find_header<'a, 'b>(headers: &'a [StrHeader<'b>], name: &str) -> Option<&'a StrHeader<'b>> { headers.iter().find(|header| header.name == name) +} + +fn retry_on_interrupt(mut do_fn: F) -> io::Result where F: FnMut() -> io::Result { + loop { + match do_fn() { + Ok(val) => return Ok(val), + Err(err) => if err.kind() != io::ErrorKind::Interrupted { + return Err(err); + }, + } + } +} + +fn create_full_path(path: &Path) -> io::Result { + if let Some(parent) = path.parent() { + try!(fs::create_dir_all(parent)); + } else { + // RFC: return an error instead? + warn!("Attempting to save file in what looks like a root directory. File path: {:?}", path); + } + + File::create(&path) } \ No newline at end of file diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index e6fa9b00a..1e9d9926b 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -16,13 +16,11 @@ extern crate buf_redux; extern crate httparse; extern crate memchr; -use mime::Mime; - use tempdir::TempDir; use std::borrow::Borrow; use std::collections::HashMap; -use std::fs::{self, File}; +use std::fs; use std::io::prelude::*; use std::path::{Path, PathBuf}; use std::{io, mem, ptr}; @@ -30,6 +28,8 @@ use std::{io, mem, ptr}; use self::boundary::BoundaryReader; use self::field::FieldHeaders; +pub use self::field::{MultipartField, MultipartFile, MultipartData, SavedFile}; + macro_rules! try_opt ( ($expr:expr) => ( match $expr { @@ -63,8 +63,6 @@ pub mod nickel; #[cfg(feature = "tiny_http")] pub mod tiny_http; -const RANDOM_FILENAME_LEN: usize = 12; - /// The server-side implementation of `multipart/form-data` requests. /// /// Implements `Borrow` to allow access to the request body, if desired. @@ -113,7 +111,7 @@ impl Multipart { return Ok(None); } - MultipartField::read_from(self) + self::field::read_field(self) } fn read_field_headers(&mut self) -> io::Result> { @@ -329,220 +327,6 @@ pub trait HttpRequest { fn body(self) -> Self::Body; } -/// A field in a multipart request. May be either text or a binary stream (file). -#[derive(Debug)] -pub struct MultipartField<'a, B: 'a> { - /// The field's name from the form - pub name: String, - /// The data of the field. Can be text or binary. - pub data: MultipartData<'a, B>, -} - -impl<'a, B: Read + 'a> MultipartField<'a, B> { - fn read_from(multipart: &'a mut Multipart) -> io::Result>> { - let field_headers = match multipart.read_field_headers() { - Ok(Some(headers)) => headers, - Ok(None) => return Ok(None), - Err(err) => return Err(err) - }; - - let data = match field_headers.cont_type { - Some(content_type) => { - MultipartData::File( - MultipartFile::from_stream( - field_headers.cont_disp.filename, - content_type.val, - &mut multipart.source, - ) - ) - }, - None => { - let text = try!(multipart.read_to_string()); - MultipartData::Text(&text) - }, - }; - - Ok(Some( - MultipartField { - name: field_headers.cont_disp.field_name, - data: data, - } - )) - } -} - -/// The data of a field in a `multipart/form-data` request. -#[derive(Debug)] -pub enum MultipartData<'a, B: 'a> { - /// The field's payload is a text string. - Text(&'a str), - /// The field's payload is a binary stream (file). - File(MultipartFile<'a, B>), - // TODO: Support multiple files per field (nested boundaries) - // MultiFiles(Vec), -} - -impl<'a, B> MultipartData<'a, B> { - /// Borrow this payload as a text field, if possible. - pub fn as_text(&self) -> Option<&str> { - match *self { - MultipartData::Text(ref s) => Some(s), - _ => None, - } - } - - /// Borrow this payload as a file field, if possible. - /// Mutably borrows so the contents can be read. - pub fn as_file(&mut self) -> Option<&mut MultipartFile<'a, B>> { - match *self { - MultipartData::File(ref mut file) => Some(file), - _ => None, - } - } -} - -/// A representation of a file in HTTP `multipart/form-data`. -/// -/// Note that the file is not yet saved to the local filesystem; -/// instead, this struct exposes `Read` and `BufRead` impls which point -/// to the beginning of the file's contents in the HTTP stream. -/// -/// You can read it to EOF, or use one of the `save_*()` methods here -/// to save it to disk. -#[derive(Debug)] -pub struct MultipartFile<'a, B: 'a> { - filename: Option, - content_type: Mime, - stream: &'a mut BoundaryReader, -} - -impl<'a, B: Read> MultipartFile<'a, B> { - fn from_stream(filename: Option, - content_type: Mime, - stream: &'a mut BoundaryReader) -> MultipartFile<'a, B> { - MultipartFile { - filename: filename, - content_type: content_type, - stream: stream, - } - } - - /// Save this file to the given output stream. - /// - /// If successful, returns the number of bytes written. - /// - /// Retries when `io::Error::kind() == io::ErrorKind::Interrupted`. - pub fn save_to(&mut self, mut out: W) -> io::Result { - retry_on_interrupt(|| io::copy(self.stream, &mut out)) - } - - /// Save this file to the given output stream, **truncated** to `limit` - /// (no more than `limit` bytes will be written out). - /// - /// If successful, returns the number of bytes written. - /// - /// Retries when `io::Error::kind() == io::ErrorKind::Interrupted`. - pub fn save_to_limited(&mut self, mut out: W, limit: u64) -> io::Result { - retry_on_interrupt(|| io::copy(&mut self.stream.take(limit), &mut out)) - } - - /// Save this file to `path`. - /// - /// Returns the saved file info on success, or any errors otherwise. - /// - /// Retries when `io::Error::kind() == io::ErrorKind::Interrupted`. - pub fn save_as>(&mut self, path: P) -> io::Result { - let path = path.into(); - let file = try!(create_full_path(&path)); - let size = try!(self.save_to(file)); - - Ok(SavedFile { - path: path, - filename: self.filename.clone(), - size: size, - }) - } - - /// Save this file in the directory pointed at by `dir`, - /// using a random alphanumeric string as the filename. - /// - /// Any missing directories in the `dir` path will be created. - /// - /// Returns the saved file's info on success, or any errors otherwise. - /// - /// Retries when `io::Error::kind() == io::ErrorKind::Interrupted`. - pub fn save_in>(&mut self, dir: P) -> io::Result { - let path = dir.as_ref().join(::random_alphanumeric(RANDOM_FILENAME_LEN)); - self.save_as(path) - } - - /// Save this file to `path`, **truncated** to `limit` (no more than `limit` bytes will be written out). - /// - /// Any missing directories in the `dir` path will be created. - /// - /// Returns the saved file's info on success, or any errors otherwise. - /// - /// Retries when `io::Error::kind() == io::ErrorKind::Interrupted`. - pub fn save_as_limited>(&mut self, path: P, limit: u64) -> io::Result { - let path = path.into(); - let file = try!(create_full_path(&path)); - let size = try!(self.save_to_limited(file, limit)); - - Ok(SavedFile { - path: path, - filename: self.filename.clone(), - size: size, - }) - } - - /// Save this file in the directory pointed at by `dir`, - /// using a random alphanumeric string as the filename. - /// - /// **Truncates** file to `limit` (no more than `limit` bytes will be written out). - /// - /// Any missing directories in the `dir` path will be created. - /// - /// Returns the saved file's info on success, or any errors otherwise. - /// - /// Retries when `io::Error::kind() == io::ErrorKind::Interrupted`. - pub fn save_in_limited>(&mut self, dir: P, limit: u64) -> io::Result { - let path = dir.as_ref().join(::random_alphanumeric(RANDOM_FILENAME_LEN)); - self.save_as_limited(path, limit) - } - - /// Get the filename of this entry, if supplied. - /// - /// ##Warning - /// You should treat this value as untrustworthy because it is an arbitrary string provided by - /// the client. You should *not* blindly append it to a directory path and save the file there, - /// as such behavior could easily be exploited by a malicious client. - pub fn filename(&self) -> Option<&str> { - self.filename.as_ref().map(String::as_ref) - } - - /// Get the MIME type (`Content-Type` value) of this file, if supplied by the client, - /// or `"applicaton/octet-stream"` otherwise. - pub fn content_type(&self) -> &Mime { - &self.content_type - } -} - -impl<'a, B: Read> Read for MultipartFile<'a, B> { - fn read(&mut self, buf: &mut [u8]) -> io::Result{ - self.stream.read(buf) - } -} - -impl<'a, B: Read> BufRead for MultipartFile<'a, B> { - fn fill_buf(&mut self) -> io::Result<&[u8]> { - self.stream.fill_buf() - } - - fn consume(&mut self, amt: usize) { - self.stream.consume(amt) - } -} - /// A result of `Multipart::save_all()`. #[derive(Debug)] pub struct Entries { @@ -653,35 +437,13 @@ impl AsRef for SaveDir { } } -/// A file saved to the local filesystem from a multipart request. -#[derive(Debug)] -pub struct SavedFile { - /// The complete path this file was saved at. - pub path: PathBuf, - - /// The original filename of this file, if one was provided in the request. - /// - /// ##Warning - /// You should treat this value as untrustworthy because it is an arbitrary string provided by - /// the client. You should *not* blindly append it to a directory path and save the file there, - /// as such behavior could easily be exploited by a malicious client. - pub filename: Option, - - /// The number of bytes written to the disk; may be truncated. - pub size: u64, +#[cfg(feature = "nightly")] +fn prepend_str(prefix: &str, mut string: String) -> String { + string.insert_str(0, prefix); + string } -fn retry_on_interrupt(mut do_fn: F) -> io::Result where F: FnMut() -> io::Result { - loop { - match do_fn() { - Ok(val) => return Ok(val), - Err(err) => if err.kind() != io::ErrorKind::Interrupted { - return Err(err); - }, - } - } -} - +#[cfg(not(feature = "nightly"))] fn prepend_str(prefix: &str, mut string: String) -> String { string.reserve(prefix.len()); @@ -701,13 +463,3 @@ fn prepend_str(prefix: &str, mut string: String) -> String { string } -fn create_full_path(path: &Path) -> io::Result { - if let Some(parent) = path.parent() { - try!(fs::create_dir_all(parent)); - } else { - // RFC: return an error instead? - warn!("Attempting to save file in what looks like a root directory. File path: {:?}", path); - } - - File::create(&path) -} From 4848f3e9b6259a02c673911a76b8cf0ae679a9c4 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Mon, 21 Nov 2016 04:44:34 -0800 Subject: [PATCH 201/453] Bump alpha version --- multipart/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 34177c552..7a963b1d3 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "multipart" -version = "0.9.0-alpha.2" +version = "0.9.0-alpha.3" authors = ["Austin Bonander "] From cea7554c393eb61892d129e859cd0d2532f11fa5 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Mon, 21 Nov 2016 04:58:25 -0800 Subject: [PATCH 202/453] Fix Iron integration assuming access to a private field of MultipartField --- multipart/src/server/iron.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/src/server/iron.rs b/multipart/src/server/iron.rs index 02310940d..fe1c4d718 100644 --- a/multipart/src/server/iron.rs +++ b/multipart/src/server/iron.rs @@ -153,7 +153,7 @@ impl Intercept { file.save_in_limited(&entries.dir, self.file_size_limit); "Error reading field: \"{}\" (filename: \"{}\")", field.name, - file.filename.as_ref().map_or("(none)", String::as_ref) + file.filename().unwrap_or("(none)") ); if file.size == self.file_size_limit { From 6c75ea79299952454cd5c085bab0bb8ab00ce375 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Mon, 21 Nov 2016 04:58:39 -0800 Subject: [PATCH 203/453] Bump alpha version (again) --- multipart/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 7a963b1d3..7047bf5c9 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "multipart" -version = "0.9.0-alpha.3" +version = "0.9.0-alpha.4" authors = ["Austin Bonander "] From d585eb3e83bff69fe7c26682fd6a5edef4b2161f Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sat, 3 Dec 2016 15:08:31 -0800 Subject: [PATCH 204/453] Fix some more logic errors in BoundaryReader, properly consume CRLF's before and after boundaries without yielding them to client code --- multipart/src/server/boundary.rs | 129 ++++++++++++++++++++----------- multipart/src/server/mod.rs | 20 +---- 2 files changed, 88 insertions(+), 61 deletions(-) diff --git a/multipart/src/server/boundary.rs b/multipart/src/server/boundary.rs index 49e50281f..b5ee2e8cd 100644 --- a/multipart/src/server/boundary.rs +++ b/multipart/src/server/boundary.rs @@ -10,6 +10,8 @@ use super::buf_redux::BufReader; use super::memchr::memchr; +use log::LogLevel; + use std::cmp; use std::borrow::Borrow; @@ -19,7 +21,7 @@ use std::io::prelude::*; /// A struct implementing `Read` and `BufRead` that will yield bytes until it sees a given sequence. #[derive(Debug)] pub struct BoundaryReader { - buf: BufReader, + source: BufReader, boundary: Vec, search_idx: usize, boundary_read: bool, @@ -30,7 +32,7 @@ impl BoundaryReader where R: Read { #[doc(hidden)] pub fn from_reader>>(reader: R, boundary: B) -> BoundaryReader { BoundaryReader { - buf: BufReader::new(reader), + source: BufReader::new(reader), boundary: boundary.into(), search_idx: 0, boundary_read: false, @@ -39,12 +41,10 @@ impl BoundaryReader where R: Read { } fn read_to_boundary(&mut self) -> io::Result<&[u8]> { - use log::LogLevel; - // Make sure there's enough bytes in the buffer to positively identify the boundary. let min_len = self.search_idx + (self.boundary.len() * 2); - let buf = try!(fill_buf_min(&mut self.buf, min_len)); + let buf = try!(fill_buf_min(&mut self.source, min_len)); if log_enabled!(LogLevel::Trace) { trace!("Buf: {:?}", String::from_utf8_lossy(buf)); @@ -55,33 +55,35 @@ impl BoundaryReader where R: Read { buf.len(), self.search_idx, self.boundary_read ); - while !(self.boundary_read || self.at_end) && self.search_idx < buf.len() { + while !self.boundary_read && self.search_idx < buf.len() { let lookahead = &buf[self.search_idx..]; - let maybe_boundary = memchr(self.boundary[0], lookahead); - - self.search_idx = match maybe_boundary { + // Find the first byte of the candidate boundary, quickly. + self.search_idx = match memchr(self.boundary[0], lookahead) { Some(boundary_start) => self.search_idx + boundary_start, None => buf.len(), }; - if self.search_idx + self.boundary.len() <= buf.len() { - let test = &buf[self.search_idx .. self.search_idx + self.boundary.len()]; - - if log_enabled!(LogLevel::Trace) { - trace!("Possible boundary: {:?}", String::from_utf8_lossy(test)); - } - - match first_nonmatching_idx(test, &self.boundary) { - Some(idx) => { - debug!("First nonmatching idx: {}", idx); - self.search_idx += idx; - }, - None => self.boundary_read = true, - } - } else { + if log_enabled!(LogLevel::Trace) { + let len = cmp::min(lookahead.len(), self.boundary.len()); + trace!("Possible boundary: {:?}", String::from_utf8_lossy(&lookahead[..len])); + } + + // See if the candidate is actually our boundary + match first_nonmatching_idx(lookahead, &self.boundary) { + Some(idx) => { + debug!("First nonmatching idx: {}", idx); + // Skip the nonmatching bit of the candidate + self.search_idx += idx; + }, + // We've only positively read the boundary if we have seen all of the candidate. + None => self.boundary_read = lookahead.len() >= self.boundary.len(), + } + + // We can still return the part of the buffer that didn't match a boundary. + if lookahead.len() < self.boundary.len() { break; - } + } } debug!( @@ -89,20 +91,21 @@ impl BoundaryReader where R: Read { buf.len(), self.search_idx, self.boundary_read ); - let mut buf_end = self.search_idx; - - if self.boundary_read && self.search_idx >= 2 { + // If the two bytes before the boundary are a CR-LF, we need to back up + // the cursor so we don't yield bytes that client code isn't expecting. + if self.search_idx >= 2 { let two_bytes_before = &buf[self.search_idx - 2 .. self.search_idx]; - debug!("Two bytes before: {:?} (\"\\r\\n\": {:?})", two_bytes_before, b"\r\n"); + debug!("Two bytes before: {:?} ({:?}) (\"\\r\\n\": {:?})", + String::from_utf8_lossy(two_bytes_before), two_bytes_before, b"\r\n"); if two_bytes_before == &*b"\r\n" { debug!("Subtract two!"); - buf_end -= 2; + self.search_idx -= 2; } } - let ret_buf = &buf[..buf_end]; + let ret_buf = &buf[..self.search_idx]; if log_enabled!(LogLevel::Trace) { trace!("Returning buf: {:?}", String::from_utf8_lossy(ret_buf)); @@ -112,9 +115,9 @@ impl BoundaryReader where R: Read { } #[doc(hidden)] - pub fn consume_boundary(&mut self) -> io::Result<()> { + pub fn consume_boundary(&mut self) -> io::Result { if self.at_end { - return Ok(()); + return Ok(true); } while !self.boundary_read { @@ -127,12 +130,46 @@ impl BoundaryReader where R: Read { self.consume(buf_len); } - self.buf.consume(self.search_idx + self.boundary.len()); - + self.source.consume(self.search_idx); self.search_idx = 0; + + if log_enabled!(LogLevel::Trace) { + trace!("Consumed up to self.search_idx, remaining buf: {:?}", + String::from_utf8_lossy(self.source.get_buf()) + ); + } + + let consume_amt = { + let buf = self.source.get_buf(); + + self.boundary.len() + if buf[..2] == *b"\r\n" { 2 } else { 0 } + }; + + self.source.consume(consume_amt); self.boundary_read = false; - - Ok(()) + + let mut bytes_after = [0, 0]; + + let read = try!(self.source.read(&mut bytes_after)); + + if read == 1 { + let _ = try!(self.source.read(&mut bytes_after[1..])); + } + + if bytes_after == *b"--" { + self.at_end = true; + } else if bytes_after != *b"\r\n" { + warn!("Unexpected bytes following boundary: {:?}", String::from_utf8_lossy(&bytes_after)); + } + + if log_enabled!(LogLevel::Trace) { + trace!("Consumed boundary (at_end: {:?}), remaining buf: {:?}", + self.at_end, + String::from_utf8_lossy(self.source.get_buf()) + ); + } + + Ok(self.at_end) } // Keeping this around to support nested boundaries later. @@ -145,7 +182,7 @@ impl BoundaryReader where R: Read { impl Borrow for BoundaryReader { fn borrow(&self) -> &R { - self.buf.get_ref() + self.source.get_ref() } } @@ -172,7 +209,7 @@ impl BufRead for BoundaryReader where R: Read { debug!("Consume! amt: {} true amt: {}", amt, true_amt); - self.buf.consume(true_amt); + self.source.consume(true_amt); self.search_idx -= true_amt; } } @@ -185,6 +222,8 @@ fn fill_buf_min(buf: &mut BufReader, min: usize) -> io::Result<&[u8] Ok(buf.get_buf()) } +// Returns the first index which `left` and `right` don't match, None otherwise. +// If they are different lengths and they match up to the end of the shorter one, returns None. fn first_nonmatching_idx(left: &[u8], right: &[u8]) -> Option { for (idx, (lb, rb)) in left.iter().zip(right).enumerate() { if lb != rb { @@ -202,8 +241,8 @@ mod test { use std::io; use std::io::prelude::*; - const BOUNDARY: &'static str = "\r\n--boundary"; - const TEST_VAL: &'static str = "\r\n--boundary\r + const BOUNDARY: &'static str = "--boundary"; + const TEST_VAL: &'static str = "--boundary\r dashed-value-1\r --boundary\r dashed-value-2\r @@ -258,7 +297,7 @@ dashed-value-2\r debug!("Testing boundary (split)"); // Substitute for `.step_by()` being unstable. - for split_at in (0 .. TEST_VAL.len()).filter(|x| x % 2 != 0) { + for split_at in 0 .. TEST_VAL.len(){ debug!("Testing split at: {}", split_at); let src = SplitReader::split(TEST_VAL.as_bytes(), split_at); @@ -281,7 +320,7 @@ dashed-value-2\r debug!("Read 2"); let _ = reader.read_to_string(buf).unwrap(); - assert_eq!(buf, "\r\ndashed-value-1"); + assert_eq!(buf, "dashed-value-1"); buf.clear(); debug!("Consume 2"); @@ -289,7 +328,7 @@ dashed-value-2\r debug!("Read 3"); let _ = reader.read_to_string(buf).unwrap(); - assert_eq!(buf, "\r\ndashed-value-2"); + assert_eq!(buf, "dashed-value-2"); buf.clear(); debug!("Consume 3"); @@ -297,6 +336,6 @@ dashed-value-2\r debug!("Read 4"); let _ = reader.read_to_string(buf).unwrap(); - assert_eq!(buf, "--"); + assert_eq!(buf, ""); } } diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index 1e9d9926b..1b36fe46c 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -107,7 +107,7 @@ impl Multipart { /// If the previously returned entry had contents of type `MultipartField::File`, /// calling this again will discard any unread contents of that entry. pub fn read_entry(&mut self) -> io::Result>> { - if !try!(self.consume_boundary()) { + if try!(self.consume_boundary()) { return Ok(None); } @@ -236,23 +236,11 @@ impl Multipart { } } + // Consume the next boundary. + // Returns `true` if the last boundary was read, `false` otherwise. fn consume_boundary(&mut self) -> io::Result { debug!("Consume boundary!"); - - try!(self.source.consume_boundary()); - - let mut out = [0; 2]; - let _ = try!(self.source.read(&mut out)); - - if *b"\r\n" == out { - Ok(true) - } else { - if *b"--" != out { - warn!("Unexpected 2-bytes after boundary: {:?}", out); - } - - Ok(false) - } + self.source.consume_boundary() } } From e70afee4b059e15ef9d4850219292363d88e0253 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sat, 3 Dec 2016 15:08:59 -0800 Subject: [PATCH 205/453] Shorten names in local_test, add some RNG to reading from the server --- multipart/src/local_test.rs | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/multipart/src/local_test.rs b/multipart/src/local_test.rs index e706b34e4..72ba8e7d0 100644 --- a/multipart/src/local_test.rs +++ b/multipart/src/local_test.rs @@ -9,7 +9,7 @@ use client::HttpStream as ClientStream; use server::HttpRequest as ServerRequest; -use rand::{self, Rng}; +use rand::{self, Rng, ThreadRng}; use std::collections::HashMap; use std::io::prelude::*; @@ -53,12 +53,12 @@ fn gen_test_fields() -> TestFields { } fn gen_string() -> String { - const MIN_LEN: usize = 3; - const MAX_LEN: usize = 8; - const MAX_DASHES: usize = 3; + const MIN_LEN: usize = 2; + const MAX_LEN: usize = 5; + const MAX_DASHES: usize = 2; - let mut rng_1 = rand::weak_rng(); - let mut rng_2 = rand::weak_rng(); + let mut rng_1 = rand::thread_rng(); + let mut rng_2 = rand::thread_rng(); let str_len_1 = rng_1.gen_range(MIN_LEN, MAX_LEN + 1); let str_len_2 = rng_2.gen_range(MIN_LEN, MAX_LEN + 1); @@ -183,6 +183,7 @@ impl HttpBuffer { data: &self.buf, boundary: &self.boundary, content_len: self.content_len, + rng: rand::thread_rng(), } } } @@ -211,16 +212,18 @@ impl ClientStream for HttpBuffer { fn finish(self) -> Result { Ok(self) } } -#[derive(Debug)] pub struct ServerBuffer<'a> { data: &'a [u8], boundary: &'a str, content_len: Option, + rng: ThreadRng, } impl<'a> Read for ServerBuffer<'a> { fn read(&mut self, out: &mut [u8]) -> io::Result { - self.data.read(out) + // Simulate the randomness of a network connection by not always reading everything + let len = self.rng.gen_range(1, out.len()); + self.data.read(&mut out[..len]) } } From 0ba0051741363eb4db8630e38a735bdb4f201b9f Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sun, 4 Dec 2016 00:33:49 -0800 Subject: [PATCH 206/453] Fixup lazy client --- multipart/src/client/lazy.rs | 114 +++++++++++++++++++---------------- 1 file changed, 61 insertions(+), 53 deletions(-) diff --git a/multipart/src/client/lazy.rs b/multipart/src/client/lazy.rs index 722446dc9..d60e5ddb6 100644 --- a/multipart/src/client/lazy.rs +++ b/multipart/src/client/lazy.rs @@ -30,6 +30,9 @@ macro_rules! try_lazy ( /// A `LazyError` wrapping `std::io::Error`. pub type LazyIoError<'a> = LazyError<'a, io::Error>; +/// `Result` type for `LazyIoError`. +pub type LazyIoResult<'a, T> = Result>; + /// An error for lazily written multipart requests, including the original error as well /// as the field which caused the error, if applicable. pub struct LazyError<'a, E> { @@ -175,7 +178,7 @@ impl<'n, 'd> Multipart<'n, 'd> { /// /// A certain amount of field data will be buffered. See /// [`prepare_threshold()`](#method.prepare_threshold) for more information on this behavior. - pub fn prepare(&mut self) -> Result, LazyIoError<'n>> { + pub fn prepare(&mut self) -> LazyIoResult<'n, PreparedFields<'d>> { self.prepare_threshold(Some(DEFAULT_BUFFER_THRESHOLD)) } @@ -186,7 +189,7 @@ impl<'n, 'd> Multipart<'n, 'd> { /// all fields are copied to memory. /// /// - pub fn prepare_threshold(&mut self, buffer_threshold: Option) -> Result, LazyIoError<'n>> { + pub fn prepare_threshold(&mut self, buffer_threshold: Option) -> LazyIoResult<'n, PreparedFields<'d>> { let boundary = super::gen_boundary(); PreparedFields::from_fields(&mut self.fields, boundary.into(), buffer_threshold) } @@ -241,8 +244,6 @@ struct Stream<'n, 'd> { /// The result of [`Multipart::prepare()`](struct.Multipart.html#method.prepare) or /// `Multipart::prepare_threshold()`. Implements `Read`, contains the entire request body. pub struct PreparedFields<'d> { - // NOTE: the order of these fields have been reversed so fields can be popped one-by-one from - // the end. fields: Vec>, #[cfg_attr(not(feature = "hyper"), allow(dead_code))] boundary: String, @@ -251,61 +252,23 @@ pub struct PreparedFields<'d> { impl<'d> PreparedFields<'d> { fn from_fields<'n>(fields: &mut Vec>, boundary: String, buffer_threshold: Option) -> Result> { - let buffer_threshold = buffer_threshold.unwrap_or(::std::u64::MAX); + let buffer_threshold = buffer_threshold.unwrap_or(0); let mut prep_fields = Vec::with_capacity(fields.len()); - // We reverse so we can pop efficiently from the end - let mut fields = fields.drain(..).rev().peekable(); + let mut fields = fields.drain(..).peekable(); let mut contiguous = Vec::new(); - let mut remainder: Option> = None; - let mut use_content_len = true; - let mut content_len = 0; while fields.peek().is_some() { - { - let mut writer = MultipartWriter::new(&mut contiguous, &*boundary); - - while let Some(field) = fields.next() { - match field.data { - Data::Text(text) => if text.len() as u64 <= buffer_threshold { - try_lazy!(field.name, writer.write_text(&field.name, &*text)); - } else { - try_lazy!(field.name, writer.write_field_headers(&field.name, None, None)); - content_len += text.len() as u64; - remainder = Some(Box::new(io::Cursor::new(CowStrAsRef(text)))); - }, - Data::File(path) => { - let (content_type, filename) = super::mime_filename(&*path); - let mut file = try_lazy!(field.name, File::open(&*path)); - let len = try_lazy!(field.name, file.metadata()).len(); - - if len <= buffer_threshold { - try_lazy!(field.name, writer.write_stream(&mut file, &field.name, filename, Some(content_type))); - } else { - try_lazy!(field.name, writer.write_field_headers(&field.name, filename, Some(content_type))); - remainder = Some(Box::new(file)); - content_len += len; - } - }, - Data::Stream(stream) => { - let filename = stream.filename.as_ref().map(|f| &**f); - try_lazy!(field.name, writer.write_field_headers(&field.name, filename, stream.content_type)); - remainder = Some(stream.stream); - use_content_len = false; - }, - } - - if remainder.is_some() { break; } - } - } - - content_len += contiguous.len() as u64; + let remainder = { + let writer = MultipartWriter::new(&mut contiguous, &*boundary); + try!(write_fields(writer, &mut fields, buffer_threshold)) + }; let contiguous = io::Cursor::new(mem::replace(&mut contiguous, Vec::new())); - if let Some(rem) = remainder.take() { + if let Some(rem) = remainder { prep_fields.push(PreparedField::Partial(contiguous.chain(rem))); } else { prep_fields.push(PreparedField::Contiguous(contiguous)); @@ -316,22 +279,21 @@ impl<'d> PreparedFields<'d> { let mut end_written = false; if let Some(&mut PreparedField::Contiguous(ref mut vec)) = prep_fields.last_mut() { - let start_len = vec.get_ref().len(); try_lazy!(write!(vec, "\r\n--{}--", boundary)); - content_len += (vec.get_ref().len() - start_len) as u64; end_written = true; } if !end_written { let vec = format!("\r\n--{}--", boundary).into_bytes(); - content_len += vec.len() as u64; prep_fields.push(PreparedField::Contiguous(io::Cursor::new(vec))); } + let content_len = get_content_len(&prep_fields); + Ok(PreparedFields { fields: prep_fields, boundary: boundary, - content_len: if use_content_len { Some(content_len) } else { None }, + content_len: content_len, }) } @@ -366,6 +328,52 @@ impl<'d> Read for PreparedFields<'d> { } } +fn get_content_len(fields: &[PreparedField]) -> Option { + let mut content_len = 0; + + for field in fields { + match *field { + PreparedField::Contiguous(ref vec) => content_len += vec.get_ref().len() as u64, + PreparedField::Partial(_) => return None, + } + } + + Some(content_len) +} + +fn write_fields<'n, 'd, I: Iterator>>(mut writer: MultipartWriter<&mut Vec>, fields: I, buffer_threshold: u64) +-> LazyIoResult<'n, Option>> { + for field in fields { + match field.data { + Data::Text(text) => if text.len() as u64 <= buffer_threshold { + try_lazy!(field.name, writer.write_text(&field.name, &*text)); + } else { + try_lazy!(field.name, writer.write_field_headers(&field.name, None, None)); + return Ok(Some(Box::new(io::Cursor::new(CowStrAsRef(text))))); + }, + Data::File(path) => { + let (content_type, filename) = super::mime_filename(&*path); + let mut file = try_lazy!(field.name, File::open(&*path)); + let len = try_lazy!(field.name, file.metadata()).len(); + + if len <= buffer_threshold { + try_lazy!(field.name, writer.write_stream(&mut file, &field.name, filename, Some(content_type))); + } else { + try_lazy!(field.name, writer.write_field_headers(&field.name, filename, Some(content_type))); + return Ok(Some(Box::new(file))); + } + }, + Data::Stream(stream) => { + let filename = stream.filename.as_ref().map(|f| &**f); + try_lazy!(field.name, writer.write_field_headers(&field.name, filename, stream.content_type)); + return Ok(Some(stream.stream)); + }, + } + } + + Ok(None) +} + #[doc(hidden)] pub enum PreparedField<'d> { Contiguous(io::Cursor>), From d35761af8388f72ab4c6c520546f0a5fe88085ed Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sun, 4 Dec 2016 00:34:07 -0800 Subject: [PATCH 207/453] Test content_len in local_test --- multipart/src/local_test.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/multipart/src/local_test.rs b/multipart/src/local_test.rs index 72ba8e7d0..953005161 100644 --- a/multipart/src/local_test.rs +++ b/multipart/src/local_test.rs @@ -97,14 +97,19 @@ fn test_client(test_fields: &TestFields) -> HttpBuffer { multipart.write_stream(file_name, &mut &**file, None, None).unwrap(); } - multipart.send().unwrap() } fn test_server(buf: HttpBuffer, mut fields: TestFields) { use server::{Multipart, MultipartData}; - let mut multipart = Multipart::from_request(buf.for_server()) + let server_buf = buf.for_server(); + + if let Some(content_len) = server_buf.content_len { + assert!(content_len == server_buf.data.len() as u64, "Supplied content_len different from actual"); + } + + let mut multipart = Multipart::from_request(server_buf) .unwrap_or_else(|_| panic!("Buffer should be multipart!")); trace!("Fields for server test: {:?}", fields); From a12aca709e07322dfcdae78d81a9031bc7aa2347 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Tue, 13 Dec 2016 21:11:57 -0800 Subject: [PATCH 208/453] Add mock types to API --- multipart/Cargo.toml | 3 +- multipart/src/lib.rs | 3 + multipart/src/local_test.rs | 149 ++++++++++++------------------------ multipart/src/mock.rs | 131 +++++++++++++++++++++++++++++++ 4 files changed, 186 insertions(+), 100 deletions(-) create mode 100644 multipart/src/mock.rs diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 7047bf5c9..f8cc9689c 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -23,7 +23,7 @@ rand = "0.3" tempdir = ">=0.3.4" #Server Dependencies -buf_redux = { version = "0.5", optional = true } +buf_redux = { version = "0.6", optional = true } memchr = { version = "0.1", optional = true } # Optional Integrations @@ -46,6 +46,7 @@ env_logger = "0.3" client = [] default = ["hyper", "server", "client"] server = ["buf_redux", "httparse", "memchr"] +mock = [] nickel_ = ["nickel", "hyper"] nightly = [] all = ["iron", "nickel_", "tiny_http"] \ No newline at end of file diff --git a/multipart/src/lib.rs b/multipart/src/lib.rs index efdffea56..3d3d8a938 100644 --- a/multipart/src/lib.rs +++ b/multipart/src/lib.rs @@ -52,6 +52,9 @@ extern crate nickel; #[cfg(feature = "tiny_http")] extern crate tiny_http; +#[cfg(any(feature = "mock", test))] +pub mod mock; + use rand::Rng; /// Chain a series of results together, with or without previous results. diff --git a/multipart/src/local_test.rs b/multipart/src/local_test.rs index 953005161..24fb516fe 100644 --- a/multipart/src/local_test.rs +++ b/multipart/src/local_test.rs @@ -4,15 +4,13 @@ // http://apache.org/licenses/LICENSE-2.0> or the MIT license , at your option. This file may not be // copied, modified, or distributed except according to those terms. -use client::HttpRequest as ClientRequest; -use client::HttpStream as ClientStream; - -use server::HttpRequest as ServerRequest; +use mock::{ClientRequest, HttpBuffer, ServerRequest}; use rand::{self, Rng, ThreadRng}; use std::collections::HashMap; use std::io::prelude::*; +use std::io::Cursor; use std::{io, iter}; #[derive(Debug)] @@ -21,16 +19,27 @@ struct TestFields { files: HashMap>, } -#[test] +//#[test] fn local_test() { - let _ = ::env_logger::init(); + do_test(test_client, "Regular"); +} + +#[test] +fn local_test_lazy() { + do_test(test_client_lazy, "Lazy"); +} + +fn do_test(client: fn(&TestFields) -> HttpBuffer, name: &str) { + let _ = ::env_logger::init(); let test_fields = gen_test_fields(); - let buf = test_client(&test_fields); + let buf = client(&test_fields); + + info!("Testing {} client", name); trace!( - "\n--Test Buffer Begin--\n{}\n--Test Buffer End--", + "\n==Test Buffer Begin==\n{}\n==Test Buffer End==", String::from_utf8_lossy(&buf.buf) ); @@ -77,7 +86,7 @@ fn gen_bytes() -> Vec { fn test_client(test_fields: &TestFields) -> HttpBuffer { use client::Multipart; - let request = MockClientRequest::default(); + let request = ClientRequest::default(); let mut test_files = test_fields.files.iter(); @@ -85,7 +94,7 @@ fn test_client(test_fields: &TestFields) -> HttpBuffer { // Intersperse file fields amongst text fields for (name, text) in &test_fields.texts { - if let Some((file_name, file)) = test_files.next() { + for (file_name, file) in &mut test_files { multipart.write_stream(file_name, &mut &**file, Some(file_name), None).unwrap(); } @@ -100,6 +109,37 @@ fn test_client(test_fields: &TestFields) -> HttpBuffer { multipart.send().unwrap() } +fn test_client_lazy(test_fields: &TestFields) -> HttpBuffer { + use client::lazy::Multipart; + + let mut multipart = Multipart::new(); + + let mut test_files = test_fields.files.iter(); + + for (name, text) in &test_fields.texts { + for (file_name, file) in &mut test_files { + multipart.add_stream(&**file_name, Cursor::new(file), Some(&**file_name), None); + } + + multipart.add_text(&**name, &**text); + } + + for (file_name, file) in test_files { + multipart.add_stream(&**file_name, Cursor::new(file), None as Option<&str>, None); + } + + let mut prepared = multipart.prepare().unwrap(); + + let mut buf = Vec::new(); + + let boundary = prepared.boundary().to_owned(); + let content_len = prepared.content_len(); + + prepared.read_to_end(&mut buf).unwrap(); + + HttpBuffer::with_buf(buf, boundary, content_len) +} + fn test_server(buf: HttpBuffer, mut fields: TestFields) { use server::{Multipart, MultipartData}; @@ -151,94 +191,5 @@ fn test_server(buf: HttpBuffer, mut fields: TestFields) { assert!(fields.files.is_empty(), "File fields were not exhausted! File fields: {:?}", fields.files); } -#[derive(Default, Debug)] -pub struct MockClientRequest { - boundary: Option, - content_len: Option, -} -impl ClientRequest for MockClientRequest { - type Stream = HttpBuffer; - type Error = io::Error; - - fn apply_headers(&mut self, boundary: &str, content_len: Option) -> bool { - self.boundary = Some(boundary.into()); - self.content_len = content_len; - true - } - - fn open_stream(self) -> Result { - debug!("MockClientRequest::open_stream called! {:?}", self); - let boundary = self.boundary.expect("HttpRequest::set_headers() was not called!"); - - Ok(HttpBuffer { buf: Vec::new(), boundary: boundary, content_len: self.content_len }) - } -} - -#[derive(Debug)] -pub struct HttpBuffer { - buf: Vec, - boundary: String, - content_len: Option, -} - -impl HttpBuffer { - pub fn for_server(&self) -> ServerBuffer { - ServerBuffer { - data: &self.buf, - boundary: &self.boundary, - content_len: self.content_len, - rng: rand::thread_rng(), - } - } -} - -impl Write for HttpBuffer { - fn write(&mut self, data: &[u8]) -> io::Result { - self.buf.write(data) - } - - fn flush(&mut self) -> io::Result<()> { - self.buf.flush() - } -} - -impl Read for HttpBuffer { - fn read(&mut self, _: &mut [u8]) -> io::Result { - unimplemented!() - } -} - -impl ClientStream for HttpBuffer { - type Request = MockClientRequest; - type Response = HttpBuffer; - type Error = io::Error; - - fn finish(self) -> Result { Ok(self) } -} - -pub struct ServerBuffer<'a> { - data: &'a [u8], - boundary: &'a str, - content_len: Option, - rng: ThreadRng, -} - -impl<'a> Read for ServerBuffer<'a> { - fn read(&mut self, out: &mut [u8]) -> io::Result { - // Simulate the randomness of a network connection by not always reading everything - let len = self.rng.gen_range(1, out.len()); - self.data.read(&mut out[..len]) - } -} - -impl<'a> ServerRequest for ServerBuffer<'a> { - type Body = Self; - - fn multipart_boundary(&self) -> Option<&str> { Some(&self.boundary) } - - fn body(self) -> Self::Body { - self - } -} diff --git a/multipart/src/mock.rs b/multipart/src/mock.rs new file mode 100644 index 000000000..80dba88d7 --- /dev/null +++ b/multipart/src/mock.rs @@ -0,0 +1,131 @@ +// Copyright 2016 `multipart` Crate Developers +// +// Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be +// copied, modified, or distributed except according to those terms. +//! +use std::io::{self, Read, Write}; +use std::fmt; + +use rand::{self, Rng, ThreadRng}; + +#[derive(Default, Debug)] +pub struct ClientRequest { + boundary: Option, + content_len: Option, +} + +#[cfg(feature = "client")] +impl ::client::HttpRequest for ClientRequest { + type Stream = HttpBuffer; + type Error = io::Error; + + fn apply_headers(&mut self, boundary: &str, content_len: Option) -> bool { + self.boundary = Some(boundary.into()); + self.content_len = content_len; + true + } + + /// ##Panics + /// If `apply_headers()` was not called. + fn open_stream(self) -> Result { + debug!("MockClientRequest::open_stream called! {:?}", self); + let boundary = self.boundary.expect("HttpRequest::set_headers() was not called!"); + + Ok(HttpBuffer::new_empty(boundary, self.content_len)) + } +} + +pub struct HttpBuffer { + pub buf: Vec, + pub boundary: String, + pub content_len: Option, + rng: ThreadRng, +} + +impl HttpBuffer { + pub fn new_empty(boundary: String, content_len: Option) -> HttpBuffer { + Self::with_buf(Vec::new(), boundary, content_len) + } + + pub fn with_buf(buf: Vec, boundary: String, content_len: Option) -> Self { + HttpBuffer { + buf: buf, + boundary: boundary, + content_len: content_len, + rng: rand::thread_rng() + } + } + + pub fn for_server(&self) -> ServerRequest { + ServerRequest { + data: &self.buf, + boundary: &self.boundary, + content_len: self.content_len, + rng: rand::thread_rng(), + } + } +} + +impl Write for HttpBuffer { + fn write(&mut self, out: &[u8]) -> io::Result { + if out.len() == 0 { + return Err(io::ErrorKind::WriteZero.into()); + } + + // Simulate the randomness of a network connection by not always reading everything + let len = self.rng.gen_range(1, out.len()); + + self.buf.write(&out[..len]) + } + + fn flush(&mut self) -> io::Result<()> { + self.buf.flush() + } +} + +#[cfg(feature = "client")] +impl ::client::HttpStream for HttpBuffer { + type Request = ClientRequest; + type Response = HttpBuffer; + type Error = io::Error; + + fn finish(self) -> Result { Ok(self) } +} + +impl fmt::Debug for HttpBuffer { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.debug_struct("multipart::mock::HttpBuffer") + .field("buf", &self.buf) + .field("boundary", &self.boundary) + .field("content_len", &self.content_len) + .finish() + } +} + +pub struct ServerRequest<'a> { + pub data: &'a [u8], + pub boundary: &'a str, + pub content_len: Option, + rng: ThreadRng, +} + +impl<'a> Read for ServerRequest<'a> { + fn read(&mut self, out: &mut [u8]) -> io::Result { + // Simulate the randomness of a network connection by not always reading everything + let len = self.rng.gen_range(1, out.len()); + self.data.read(&mut out[..len]) + } +} + +#[cfg(feature = "server")] +impl<'a> ::server::HttpRequest for ServerRequest<'a> { + type Body = Self; + + fn multipart_boundary(&self) -> Option<&str> { Some(&self.boundary) } + + fn body(self) -> Self::Body { + self + } +} \ No newline at end of file From 9fcbe2d677e9c71aeb82609cee3df80d34649e11 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Tue, 13 Dec 2016 21:14:41 -0800 Subject: [PATCH 209/453] Remove unused imports in local_test --- multipart/src/local_test.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/multipart/src/local_test.rs b/multipart/src/local_test.rs index 24fb516fe..95b749228 100644 --- a/multipart/src/local_test.rs +++ b/multipart/src/local_test.rs @@ -4,14 +4,14 @@ // http://apache.org/licenses/LICENSE-2.0> or the MIT license , at your option. This file may not be // copied, modified, or distributed except according to those terms. -use mock::{ClientRequest, HttpBuffer, ServerRequest}; +use mock::{ClientRequest, HttpBuffer}; -use rand::{self, Rng, ThreadRng}; +use rand::{self, Rng}; use std::collections::HashMap; use std::io::prelude::*; use std::io::Cursor; -use std::{io, iter}; +use std::iter; #[derive(Debug)] struct TestFields { From caa4f5ec47f4f7872233103ac4c05f8fff74f96e Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Wed, 14 Dec 2016 01:31:26 -0800 Subject: [PATCH 210/453] WIP trying to get lazy client test working --- multipart/Cargo.toml | 5 ++- multipart/src/client/lazy.rs | 49 +++++++++++----------- multipart/src/client/mod.rs | 18 ++++++-- multipart/src/server/boundary.rs | 70 ++++++++++++-------------------- multipart/src/server/mod.rs | 45 ++++---------------- 5 files changed, 76 insertions(+), 111 deletions(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index f8cc9689c..10dedfbf7 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -24,7 +24,8 @@ tempdir = ">=0.3.4" #Server Dependencies buf_redux = { version = "0.6", optional = true } -memchr = { version = "0.1", optional = true } +safemem = { version = "0.2", optional = true } +twoway = { version = "0.1", optional = true } # Optional Integrations hyper = { version = "0.9", optional = true } @@ -45,7 +46,7 @@ env_logger = "0.3" [features] client = [] default = ["hyper", "server", "client"] -server = ["buf_redux", "httparse", "memchr"] +server = ["buf_redux", "httparse", "safemem", "twoway"] mock = [] nickel_ = ["nickel", "hyper"] nightly = [] diff --git a/multipart/src/client/lazy.rs b/multipart/src/client/lazy.rs index d60e5ddb6..d4547d8c7 100644 --- a/multipart/src/client/lazy.rs +++ b/multipart/src/client/lazy.rs @@ -8,7 +8,7 @@ use std::fs::File; use std::path::{Path, PathBuf}; use std::io::prelude::*; -use std::{fmt, io, mem}; +use std::{fmt, io, mem, vec}; use super::{HttpRequest, HttpStream, MultipartWriter}; @@ -244,7 +244,8 @@ struct Stream<'n, 'd> { /// The result of [`Multipart::prepare()`](struct.Multipart.html#method.prepare) or /// `Multipart::prepare_threshold()`. Implements `Read`, contains the entire request body. pub struct PreparedFields<'d> { - fields: Vec>, + fields: vec::IntoIter>, + next_field: Option>, #[cfg_attr(not(feature = "hyper"), allow(dead_code))] boundary: String, content_len: Option, @@ -291,7 +292,8 @@ impl<'d> PreparedFields<'d> { let content_len = get_content_len(&prep_fields); Ok(PreparedFields { - fields: prep_fields, + fields: prep_fields.into_iter(), + next_field: None, boundary: boundary, content_len: content_len, }) @@ -311,20 +313,29 @@ impl<'d> PreparedFields<'d> { impl<'d> Read for PreparedFields<'d> { fn read(&mut self, buf: &mut [u8]) -> io::Result { - if buf.is_empty() { return Ok(0) } - if self.fields.is_empty() { return Ok(0) } + if buf.len() == 0 { return Err(io::ErrorKind::WriteZero.into()) } - let bytes_read = if let Some(mut curr) = self.fields.last_mut() { - try!(curr.read(buf)) - } else { - 0 - }; + let mut total_read = 0; + + while total_read < buf.len() { + if let None = self.next_field { + self.next_field = self.fields.next(); + } + + let buf = &mut buf[total_read..]; + + let num_read = if let Some(ref mut field) = self.next_field { + try!(field.read(buf)) + } else { + break; + }; - if bytes_read == 0 { - let _ = self.fields.pop(); + if num_read == 0 { + self.next_field = None; + } } - Ok(bytes_read) + Ok(total_read) } } @@ -477,17 +488,7 @@ mod hyper { impl<'d> super::PreparedFields<'d> { /// #### Feature: `hyper` /// Convert `self` to `hyper::client::Body`. - pub fn to_body<'b>(&'b mut self) -> Body<'b> { - use super::PreparedField; - // We have a single contiguous body, provide it directly - if self.fields.len() == 1 { - if let PreparedField::Contiguous(ref body) = self.fields[0] { - return Body::BufBody(body.get_ref(), body.get_ref().len()); - } else { - unreachable!("Only one field but it was not contiguous!"); - } - } - + pub fn to_body<'b>(&'b mut self) -> Body<'b> where 'd: 'b { if let Some(content_len) = self.content_len { Body::SizedBody(self, content_len) } else { diff --git a/multipart/src/client/mod.rs b/multipart/src/client/mod.rs index 22dc9e9b3..2aaea4809 100644 --- a/multipart/src/client/mod.rs +++ b/multipart/src/client/mod.rs @@ -206,7 +206,11 @@ impl<'a, W: Write> MultipartWriter<'a, W> { } fn write_boundary(&mut self) -> io::Result<()> { - write!(self.inner, "\r\n--{}\r\n", self.boundary) + if self.data_written { + try!(self.inner.write_all(b"\r\n")); + } + + write!(self.inner, "--{}\r\n", self.boundary) } fn write_text(&mut self, name: &str, text: &str) -> io::Result<()> { @@ -235,7 +239,7 @@ impl<'a, W: Write> MultipartWriter<'a, W> { fn write_field_headers(&mut self, name: &str, filename: Option<&str>, content_type: Option) -> io::Result<()> { - chain_result! { + let res = chain_result! { // Write the first boundary, or the boundary for the previous field. self.write_boundary(), { self.data_written = true; Ok(()) }, @@ -245,7 +249,15 @@ impl<'a, W: Write> MultipartWriter<'a, W> { content_type.map(|content_type| write!(self.inner, "\r\nContent-Type: {}", content_type)) .unwrap_or(Ok(())), self.inner.write_all(b"\r\n\r\n") - } + }; + + self.data_written = true; + + res + } + + fn inner_mut(&mut self) -> &mut W { + &mut self.inner } fn finish(mut self) -> io::Result { diff --git a/multipart/src/server/boundary.rs b/multipart/src/server/boundary.rs index b5ee2e8cd..d9b4de56c 100644 --- a/multipart/src/server/boundary.rs +++ b/multipart/src/server/boundary.rs @@ -8,7 +8,7 @@ //! Boundary parsing for `multipart` requests. use super::buf_redux::BufReader; -use super::memchr::memchr; +use super::{safemem, twoway}; use log::LogLevel; @@ -31,9 +31,12 @@ pub struct BoundaryReader { impl BoundaryReader where R: Read { #[doc(hidden)] pub fn from_reader>>(reader: R, boundary: B) -> BoundaryReader { + let mut boundary = boundary.into(); + safemem::prepend(b"--", &mut boundary); + BoundaryReader { source: BufReader::new(reader), - boundary: boundary.into(), + boundary: boundary, search_idx: 0, boundary_read: false, at_end: false, @@ -45,7 +48,9 @@ impl BoundaryReader where R: Read { let min_len = self.search_idx + (self.boundary.len() * 2); let buf = try!(fill_buf_min(&mut self.source, min_len)); - + + self.at_end = buf.len() == 0; + if log_enabled!(LogLevel::Trace) { trace!("Buf: {:?}", String::from_utf8_lossy(buf)); } @@ -55,34 +60,20 @@ impl BoundaryReader where R: Read { buf.len(), self.search_idx, self.boundary_read ); - while !self.boundary_read && self.search_idx < buf.len() { + if !self.boundary_read && self.search_idx < buf.len() { let lookahead = &buf[self.search_idx..]; - // Find the first byte of the candidate boundary, quickly. - self.search_idx = match memchr(self.boundary[0], lookahead) { - Some(boundary_start) => self.search_idx + boundary_start, - None => buf.len(), - }; + debug!("Find boundary loop! Lookahead len: {}", lookahead.len()); - if log_enabled!(LogLevel::Trace) { - let len = cmp::min(lookahead.len(), self.boundary.len()); - trace!("Possible boundary: {:?}", String::from_utf8_lossy(&lookahead[..len])); - } - - // See if the candidate is actually our boundary - match first_nonmatching_idx(lookahead, &self.boundary) { - Some(idx) => { - debug!("First nonmatching idx: {}", idx); - // Skip the nonmatching bit of the candidate - self.search_idx += idx; + // Look for the boundary, or if it isn't found, stop near the end. + match twoway::find_bytes(lookahead, &self.boundary) { + Some(found_idx) => { + self.search_idx += found_idx; + self.boundary_read = true; }, - // We've only positively read the boundary if we have seen all of the candidate. - None => self.boundary_read = lookahead.len() >= self.boundary.len(), - } - - // We can still return the part of the buffer that didn't match a boundary. - if lookahead.len() < self.boundary.len() { - break; + None => { + self.search_idx += lookahead.len().saturating_sub(self.boundary.len() + 2); + } } } @@ -93,7 +84,7 @@ impl BoundaryReader where R: Read { // If the two bytes before the boundary are a CR-LF, we need to back up // the cursor so we don't yield bytes that client code isn't expecting. - if self.search_idx >= 2 { + if self.boundary_read && self.search_idx >= 2 { let two_bytes_before = &buf[self.search_idx - 2 .. self.search_idx]; debug!("Two bytes before: {:?} ({:?}) (\"\\r\\n\": {:?})", @@ -120,9 +111,13 @@ impl BoundaryReader where R: Read { return Ok(true); } - while !self.boundary_read { + while !(self.boundary_read || self.at_end){ + debug!("Boundary not found yet"); + let buf_len = try!(self.read_to_boundary()).len(); + debug!("Discarding {} bytes", buf_len); + if buf_len == 0 { break; } @@ -141,8 +136,7 @@ impl BoundaryReader where R: Read { let consume_amt = { let buf = self.source.get_buf(); - - self.boundary.len() + if buf[..2] == *b"\r\n" { 2 } else { 0 } + self.boundary.len() + if buf.len() >=2 && buf[..2] == *b"\r\n" { 2 } else { 0 } }; self.source.consume(consume_amt); @@ -222,18 +216,6 @@ fn fill_buf_min(buf: &mut BufReader, min: usize) -> io::Result<&[u8] Ok(buf.get_buf()) } -// Returns the first index which `left` and `right` don't match, None otherwise. -// If they are different lengths and they match up to the end of the shorter one, returns None. -fn first_nonmatching_idx(left: &[u8], right: &[u8]) -> Option { - for (idx, (lb, rb)) in left.iter().zip(right).enumerate() { - if lb != rb { - return Some(idx); - } - } - - None -} - #[cfg(test)] mod test { use super::BoundaryReader; @@ -241,7 +223,7 @@ mod test { use std::io; use std::io::prelude::*; - const BOUNDARY: &'static str = "--boundary"; + const BOUNDARY: &'static str = "boundary"; const TEST_VAL: &'static str = "--boundary\r dashed-value-1\r --boundary\r diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index 1b36fe46c..3470bff38 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -14,7 +14,8 @@ extern crate buf_redux; extern crate httparse; -extern crate memchr; +extern crate safemem; +extern crate twoway; use tempdir::TempDir; @@ -23,7 +24,7 @@ use std::collections::HashMap; use std::fs; use std::io::prelude::*; use std::path::{Path, PathBuf}; -use std::{io, mem, ptr}; +use std::{io, mem}; use self::boundary::BoundaryReader; use self::field::FieldHeaders; @@ -88,14 +89,9 @@ impl Multipart<()> { impl Multipart { /// Construct a new `Multipart` with the given body reader and boundary. - /// This will prepend the requisite `"--"` to the boundary. pub fn with_body>(body: B, boundary: Bnd) -> Self { - let boundary = prepend_str("--", boundary.into()); - - debug!("Boundary: {}", boundary); - Multipart { - source: BoundaryReader::from_reader(body, boundary), + source: BoundaryReader::from_reader(body, boundary.into()), line_buf: String::new(), } } @@ -236,8 +232,8 @@ impl Multipart { } } - // Consume the next boundary. - // Returns `true` if the last boundary was read, `false` otherwise. + /// Consume the next boundary. + /// Returns `true` if the last boundary was read, `false` otherwise. fn consume_boundary(&mut self) -> io::Result { debug!("Consume boundary!"); self.source.consume_boundary() @@ -423,31 +419,4 @@ impl AsRef for SaveDir { fn as_ref(&self) -> &Path { self.as_path() } -} - -#[cfg(feature = "nightly")] -fn prepend_str(prefix: &str, mut string: String) -> String { - string.insert_str(0, prefix); - string -} - -#[cfg(not(feature = "nightly"))] -fn prepend_str(prefix: &str, mut string: String) -> String { - string.reserve(prefix.len()); - - unsafe { - let bytes = string.as_mut_vec(); - - // This addition is safe because it was already done in `String::reserve()` - // which would have panicked if it overflowed. - let old_len = bytes.len(); - let new_len = bytes.len() + prefix.len(); - bytes.set_len(new_len); - - ptr::copy(bytes.as_ptr(), bytes[prefix.len()..].as_mut_ptr(), old_len); - ptr::copy(prefix.as_ptr(), bytes.as_mut_ptr(), prefix.len()); - } - - string -} - +} \ No newline at end of file From 23a08dbf372b3b8025cc1b30c8cb3c5c6fc1b95d Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Wed, 14 Dec 2016 04:37:07 -0800 Subject: [PATCH 211/453] Fix faulty assumption of `Rng::gen_range` --- multipart/src/mock.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/multipart/src/mock.rs b/multipart/src/mock.rs index 80dba88d7..562b9a97e 100644 --- a/multipart/src/mock.rs +++ b/multipart/src/mock.rs @@ -75,7 +75,7 @@ impl Write for HttpBuffer { } // Simulate the randomness of a network connection by not always reading everything - let len = self.rng.gen_range(1, out.len()); + let len = self.rng.gen_range(1, out.len() + 1); self.buf.write(&out[..len]) } @@ -113,8 +113,12 @@ pub struct ServerRequest<'a> { impl<'a> Read for ServerRequest<'a> { fn read(&mut self, out: &mut [u8]) -> io::Result { + if out.len() == 0 { + return Err(io::ErrorKind::WriteZero.into()); + } + // Simulate the randomness of a network connection by not always reading everything - let len = self.rng.gen_range(1, out.len()); + let len = self.rng.gen_range(1, out.len() + 1); self.data.read(&mut out[..len]) } } From 1ef9c9a46083630263c72818b0c00e7bfcd30795 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Wed, 14 Dec 2016 04:37:30 -0800 Subject: [PATCH 212/453] Remove redundant write --- multipart/src/client/mod.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/multipart/src/client/mod.rs b/multipart/src/client/mod.rs index 2aaea4809..1dfcf06a0 100644 --- a/multipart/src/client/mod.rs +++ b/multipart/src/client/mod.rs @@ -251,8 +251,6 @@ impl<'a, W: Write> MultipartWriter<'a, W> { self.inner.write_all(b"\r\n\r\n") }; - self.data_written = true; - res } From dd18f888a3faf4c278cc3dcd849501aef24927c0 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Wed, 14 Dec 2016 04:37:53 -0800 Subject: [PATCH 213/453] Fix lazy client and local_test --- multipart/src/client/lazy.rs | 120 ++++++++++++++++++++--------------- multipart/src/local_test.rs | 12 ++-- 2 files changed, 74 insertions(+), 58 deletions(-) diff --git a/multipart/src/client/lazy.rs b/multipart/src/client/lazy.rs index d4547d8c7..ce4be5067 100644 --- a/multipart/src/client/lazy.rs +++ b/multipart/src/client/lazy.rs @@ -1,5 +1,7 @@ //! Multipart requests which write out their data in one fell swoop. +use log::LogLevel; + use mime::Mime; use std::borrow::Cow; @@ -8,6 +10,7 @@ use std::fs::File; use std::path::{Path, PathBuf}; use std::io::prelude::*; +use std::io::Cursor; use std::{fmt, io, mem, vec}; use super::{HttpRequest, HttpStream, MultipartWriter}; @@ -163,7 +166,7 @@ impl<'n, 'd> Multipart<'n, 'd> { /// Convert `req` to `HttpStream`, write out the fields in this request, and finish the /// request, returning the response if successful, or the first error encountered. - pub fn send(&mut self, req: R) -> Result<::Response, LazyError<'n, ::Error>> { + pub fn send(&mut self, req: R) -> Result<< R::Stream as HttpStream >::Response, LazyError<'n, < R::Stream as HttpStream >::Error>> { let (boundary, stream) = try_lazy!(super::open_stream(req, None)); let mut writer = MultipartWriter::new(stream, boundary); @@ -179,7 +182,7 @@ impl<'n, 'd> Multipart<'n, 'd> { /// A certain amount of field data will be buffered. See /// [`prepare_threshold()`](#method.prepare_threshold) for more information on this behavior. pub fn prepare(&mut self) -> LazyIoResult<'n, PreparedFields<'d>> { - self.prepare_threshold(Some(DEFAULT_BUFFER_THRESHOLD)) + self.prepare_threshold(Some(DEFAULT_BUFFER_THRESHOLD)) } /// Export the multipart data contained in this lazy request to an adaptor which implements `Read`. @@ -187,8 +190,6 @@ impl<'n, 'd> Multipart<'n, 'd> { /// #### Buffering /// For efficiency, text and file fields smaller than `buffer_threshold` are copied to an in-memory buffer. If `None`, /// all fields are copied to memory. - /// - /// pub fn prepare_threshold(&mut self, buffer_threshold: Option) -> LazyIoResult<'n, PreparedFields<'d>> { let boundary = super::gen_boundary(); PreparedFields::from_fields(&mut self.fields, boundary.into(), buffer_threshold) @@ -208,10 +209,10 @@ impl<'n, 'd> Field<'n, 'd> { match self.data { Data::Text(ref text) => writer.write_text(&self.name, text), Data::File(ref path) => writer.write_file(&self.name, path), - Data::Stream(ref mut stream) => + Data::Stream(ref mut stream) => writer.write_stream( - &mut stream.stream, - &self.name, + &mut stream.stream, + &self.name, stream.filename.as_ref().map(|f| &**f), stream.content_type.clone(), ), @@ -226,8 +227,8 @@ enum Data<'n, 'd> { } impl<'n, 'd> fmt::Debug for Data<'n, 'd> { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - match *self { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match *self { Data::Text(ref text) => write!(f, "Data::Text({:?})", text), Data::File(ref path) => write!(f, "Data::File({:?})", path), Data::Stream(_) => f.write_str("Data::Stream(Box"), @@ -246,51 +247,39 @@ struct Stream<'n, 'd> { pub struct PreparedFields<'d> { fields: vec::IntoIter>, next_field: Option>, - #[cfg_attr(not(feature = "hyper"), allow(dead_code))] boundary: String, content_len: Option, } impl<'d> PreparedFields<'d> { fn from_fields<'n>(fields: &mut Vec>, boundary: String, buffer_threshold: Option) -> Result> { - let buffer_threshold = buffer_threshold.unwrap_or(0); + let buffer_threshold = buffer_threshold.unwrap_or(::std::u64::MAX); - let mut prep_fields = Vec::with_capacity(fields.len()); + debug!("Field count: {}", fields.len()); - let mut fields = fields.drain(..).peekable(); - - let mut contiguous = Vec::new(); + let mut prep_fields = Vec::new(); - while fields.peek().is_some() { - let remainder = { - let writer = MultipartWriter::new(&mut contiguous, &*boundary); - try!(write_fields(writer, &mut fields, buffer_threshold)) - }; + let mut fields = fields.drain(..).peekable(); - let contiguous = io::Cursor::new(mem::replace(&mut contiguous, Vec::new())); + { + let mut writer = MultipartWriter::new(Vec::new(), &*boundary); - if let Some(rem) = remainder { - prep_fields.push(PreparedField::Partial(contiguous.chain(rem))); - } else { - prep_fields.push(PreparedField::Contiguous(contiguous)); + while fields.peek().is_some() { + if let Some(rem) = try!(write_fields(&mut writer, &mut fields, buffer_threshold)) { + let contiguous = mem::replace(writer.inner_mut(), Vec::new()); + prep_fields.push(PreparedField::Partial(Cursor::new(contiguous), rem)); + } } - } - // FIXME: when non-lexical borrow scopes land, convert this to a single if-let/else - let mut end_written = false; - - if let Some(&mut PreparedField::Contiguous(ref mut vec)) = prep_fields.last_mut() { - try_lazy!(write!(vec, "\r\n--{}--", boundary)); - end_written = true; - } - - if !end_written { - let vec = format!("\r\n--{}--", boundary).into_bytes(); - prep_fields.push(PreparedField::Contiguous(io::Cursor::new(vec))); + let contiguous = writer.finish().unwrap(); + + prep_fields.push(PreparedField::Contiguous(Cursor::new(contiguous))); } let content_len = get_content_len(&prep_fields); + debug!("Prepared fields len: {}", prep_fields.len()); + Ok(PreparedFields { fields: prep_fields.into_iter(), next_field: None, @@ -308,7 +297,7 @@ impl<'d> PreparedFields<'d> { /// Get the boundary that was used to serialize the request. pub fn boundary(&self) -> &str { &self.boundary - } + } } impl<'d> Read for PreparedFields<'d> { @@ -333,6 +322,8 @@ impl<'d> Read for PreparedFields<'d> { if num_read == 0 { self.next_field = None; } + + total_read += num_read; } Ok(total_read) @@ -345,18 +336,18 @@ fn get_content_len(fields: &[PreparedField]) -> Option { for field in fields { match *field { PreparedField::Contiguous(ref vec) => content_len += vec.get_ref().len() as u64, - PreparedField::Partial(_) => return None, + PreparedField::Partial(..) => return None, } } Some(content_len) } -fn write_fields<'n, 'd, I: Iterator>>(mut writer: MultipartWriter<&mut Vec>, fields: I, buffer_threshold: u64) --> LazyIoResult<'n, Option>> { +fn write_fields<'n, 'd, I: Iterator>>(writer: &mut MultipartWriter>, fields: I, buffer_threshold: u64) + -> LazyIoResult<'n, Option>> { for field in fields { match field.data { - Data::Text(text) => if text.len() as u64 <= buffer_threshold { + Data::Text(text) => if text.len() as u64 <= buffer_threshold { try_lazy!(field.name, writer.write_text(&field.name, &*text)); } else { try_lazy!(field.name, writer.write_field_headers(&field.name, None, None)); @@ -370,13 +361,13 @@ fn write_fields<'n, 'd, I: Iterator>>(mut writer: Multipart if len <= buffer_threshold { try_lazy!(field.name, writer.write_stream(&mut file, &field.name, filename, Some(content_type))); } else { - try_lazy!(field.name, writer.write_field_headers(&field.name, filename, Some(content_type))); return Ok(Some(Box::new(file))); } }, Data::Stream(stream) => { + let content_type = stream.content_type.or_else(|| Some(::mime_guess::octet_stream())); let filename = stream.filename.as_ref().map(|f| &**f); - try_lazy!(field.name, writer.write_field_headers(&field.name, filename, stream.content_type)); + try_lazy!(field.name, writer.write_field_headers(&field.name, filename, content_type)); return Ok(Some(stream.stream)); }, } @@ -388,14 +379,40 @@ fn write_fields<'n, 'd, I: Iterator>>(mut writer: Multipart #[doc(hidden)] pub enum PreparedField<'d> { Contiguous(io::Cursor>), - Partial(io::Chain>, Box>), + Partial(io::Cursor>, Box), } impl<'d> Read for PreparedField<'d> { fn read(&mut self, buf: &mut [u8]) -> io::Result { match *self { PreparedField::Contiguous(ref mut vec) => vec.read(buf), - PreparedField::Partial(ref mut chain) => chain.read(buf), + PreparedField::Partial(ref mut cursor, ref mut remainder) => { + match cursor.read(buf) { + Ok(0) => remainder.read(buf), + res => res, + } + }, + } + } +} + +impl<'d> fmt::Debug for PreparedField<'d> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match *self { + PreparedField::Contiguous(ref bytes) => { + if log_enabled!(LogLevel::Trace) { + write!(f, "PreparedField::Contiguous(\n{:?}\n)", String::from_utf8_lossy(bytes.get_ref())) + } else { + write!(f, "PreparedField::Contiguous(len: {})", bytes.get_ref().len()) + } + }, + PreparedField::Partial(ref bytes, _) => { + if log_enabled!(LogLevel::Trace) { + write!(f, "PreparedField::Partial(\n{:?}\n, Box)", String::from_utf8_lossy(bytes.get_ref())) + } else { + write!(f, "PreparedField::Partial(len: {}, Box)", bytes.get_ref().len()) + } + } } } } @@ -457,7 +474,7 @@ mod hyper { /// Supplies the fields in the body, optionally setting the content-length header if /// applicable (all added fields were text or files, i.e. no streams). pub fn client_request(&mut self, client: &Client, url: U) -> HyperResult { - self.client_request_mut(client, url, |r| r) + self.client_request_mut(client, url, |r| r) } /// #### Feature: `hyper` @@ -467,8 +484,7 @@ mod hyper { /// Note that the body, and the `ContentType` and `ContentLength` headers will be /// overwritten, either by this method or by Hyper. pub fn client_request_mut RequestBuilder>(&mut self, client: &Client, url: U, - mut_fn: F) -> HyperResult { - + mut_fn: F) -> HyperResult { let mut fields = match self.prepare() { Ok(fields) => fields, Err(err) => { @@ -476,12 +492,12 @@ mod hyper { return Err(err.error.into()); }, }; - - + + mut_fn(client.post(url)) .header(::client::hyper::content_type(fields.boundary())) .body(fields.to_body()) - .send() + .send() } } diff --git a/multipart/src/local_test.rs b/multipart/src/local_test.rs index 95b749228..cf82f47a6 100644 --- a/multipart/src/local_test.rs +++ b/multipart/src/local_test.rs @@ -19,7 +19,7 @@ struct TestFields { files: HashMap>, } -//#[test] +#[test] fn local_test() { do_test(test_client, "Regular"); } @@ -32,11 +32,13 @@ fn local_test_lazy() { fn do_test(client: fn(&TestFields) -> HttpBuffer, name: &str) { let _ = ::env_logger::init(); + info!("Testing {} client", name); + let test_fields = gen_test_fields(); - let buf = client(&test_fields); + trace!("Fields for test: {:?}", test_fields); - info!("Testing {} client", name); + let buf = client(&test_fields); trace!( "\n==Test Buffer Begin==\n{}\n==Test Buffer End==", @@ -128,7 +130,7 @@ fn test_client_lazy(test_fields: &TestFields) -> HttpBuffer { multipart.add_stream(&**file_name, Cursor::new(file), None as Option<&str>, None); } - let mut prepared = multipart.prepare().unwrap(); + let mut prepared = multipart.prepare_threshold(None).unwrap(); let mut buf = Vec::new(); @@ -152,8 +154,6 @@ fn test_server(buf: HttpBuffer, mut fields: TestFields) { let mut multipart = Multipart::from_request(server_buf) .unwrap_or_else(|_| panic!("Buffer should be multipart!")); - trace!("Fields for server test: {:?}", fields); - while let Ok(Some(mut field)) = multipart.read_entry() { match field.data { MultipartData::Text(text) => { From d91d141f6b1c0761400837f8e70d21b360b360f3 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Wed, 14 Dec 2016 04:38:20 -0800 Subject: [PATCH 214/453] Bump version --- multipart/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 10dedfbf7..dfac3b379 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "multipart" -version = "0.9.0-alpha.4" +version = "0.9.0-alpha.5" authors = ["Austin Bonander "] From 4969715802aa49ec65118e04995404fa35931482 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Wed, 14 Dec 2016 15:16:40 -0800 Subject: [PATCH 215/453] Fixed usage of io::ErrorKind::into(), bumped version --- multipart/Cargo.toml | 2 +- multipart/src/client/lazy.rs | 4 +++- multipart/src/lib.rs | 3 +-- multipart/src/mock.rs | 6 ++++-- 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index dfac3b379..fe9c51396 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "multipart" -version = "0.9.0-alpha.5" +version = "0.9.0-alpha.6" authors = ["Austin Bonander "] diff --git a/multipart/src/client/lazy.rs b/multipart/src/client/lazy.rs index ce4be5067..9f06c4679 100644 --- a/multipart/src/client/lazy.rs +++ b/multipart/src/client/lazy.rs @@ -302,7 +302,9 @@ impl<'d> PreparedFields<'d> { impl<'d> Read for PreparedFields<'d> { fn read(&mut self, buf: &mut [u8]) -> io::Result { - if buf.len() == 0 { return Err(io::ErrorKind::WriteZero.into()) } + if buf.len() == 0 { + return Ok(0); + } let mut total_read = 0; diff --git a/multipart/src/lib.rs b/multipart/src/lib.rs index 3d3d8a938..5eb4c7578 100644 --- a/multipart/src/lib.rs +++ b/multipart/src/lib.rs @@ -102,5 +102,4 @@ mod local_test; fn random_alphanumeric(len: usize) -> String { rand::thread_rng().gen_ascii_chars().take(len).collect() -} - +} \ No newline at end of file diff --git a/multipart/src/mock.rs b/multipart/src/mock.rs index 562b9a97e..9733fb04a 100644 --- a/multipart/src/mock.rs +++ b/multipart/src/mock.rs @@ -71,7 +71,8 @@ impl HttpBuffer { impl Write for HttpBuffer { fn write(&mut self, out: &[u8]) -> io::Result { if out.len() == 0 { - return Err(io::ErrorKind::WriteZero.into()); + debug!("Passed a zero-sized buffer."); + return Ok(0); } // Simulate the randomness of a network connection by not always reading everything @@ -114,7 +115,8 @@ pub struct ServerRequest<'a> { impl<'a> Read for ServerRequest<'a> { fn read(&mut self, out: &mut [u8]) -> io::Result { if out.len() == 0 { - return Err(io::ErrorKind::WriteZero.into()); + debug!("Passed a zero-sized buffer."); + return Ok(0); } // Simulate the randomness of a network connection by not always reading everything From 1cc8e3083e724af2ff3d72df9c4d85dd58821755 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Wed, 14 Dec 2016 15:55:46 -0800 Subject: [PATCH 216/453] Make debug messages for zero-sized reads more useful --- multipart/src/client/lazy.rs | 1 + multipart/src/mock.rs | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/multipart/src/client/lazy.rs b/multipart/src/client/lazy.rs index 9f06c4679..c60af8cc6 100644 --- a/multipart/src/client/lazy.rs +++ b/multipart/src/client/lazy.rs @@ -303,6 +303,7 @@ impl<'d> PreparedFields<'d> { impl<'d> Read for PreparedFields<'d> { fn read(&mut self, buf: &mut [u8]) -> io::Result { if buf.len() == 0 { + debug!("PreparedFields::read() was passed a zero-sized buffer."); return Ok(0); } diff --git a/multipart/src/mock.rs b/multipart/src/mock.rs index 9733fb04a..21e201b58 100644 --- a/multipart/src/mock.rs +++ b/multipart/src/mock.rs @@ -71,7 +71,7 @@ impl HttpBuffer { impl Write for HttpBuffer { fn write(&mut self, out: &[u8]) -> io::Result { if out.len() == 0 { - debug!("Passed a zero-sized buffer."); + debug!("HttpBuffer::write() was passed a zero-sized buffer."); return Ok(0); } @@ -115,7 +115,7 @@ pub struct ServerRequest<'a> { impl<'a> Read for ServerRequest<'a> { fn read(&mut self, out: &mut [u8]) -> io::Result { if out.len() == 0 { - debug!("Passed a zero-sized buffer."); + debug!("ServerRequest::read() was passed a zero-sized buffer."); return Ok(0); } From bb45052fed425d580856e60d7d86b9e13af58b33 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Thu, 15 Dec 2016 18:16:55 -0800 Subject: [PATCH 217/453] Add random reads to test program --- multipart/src/bin/test_multipart.rs | 32 ++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/multipart/src/bin/test_multipart.rs b/multipart/src/bin/test_multipart.rs index 79a353fb1..b3c0119ce 100644 --- a/multipart/src/bin/test_multipart.rs +++ b/multipart/src/bin/test_multipart.rs @@ -1,13 +1,16 @@ +#[macro_use] extern crate log; extern crate multipart; - -extern crate log; +extern crate rand; use log::{LogRecord, LogMetadata, LogLevelFilter}; use multipart::server::Multipart; +use rand::{Rng, ThreadRng}; + use std::fs::File; use std::env; +use std::io::{self, Read}; const LOG_LEVEL: LogLevelFilter = LogLevelFilter::Debug; @@ -40,11 +43,34 @@ fn main() { let file = File::open(file).expect("Could not open file"); - let mut multipart = Multipart::with_body(file, boundary); + let reader = RandomReader { + inner: file, + rng: rand::thread_rng() + }; + + let mut multipart = Multipart::with_body(reader, boundary); while let Some(field) = multipart.read_entry().unwrap() { println!("Read field: {:?}", field.name); } println!("All entries read!"); +} + +struct RandomReader { + inner: R, + rng: ThreadRng, +} + +impl Read for RandomReader { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + if buf.len() == 0 { + debug!("RandomReader::read() passed a zero-sized buffer."); + return Ok(0); + } + + let len = self.rng.gen_range(1, buf.len() + 1); + + self.inner.read(&mut buf[..len]) + } } \ No newline at end of file From 15d3c07910d8ab7d64f2ba87c24bc15eeea9d5e0 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Thu, 15 Dec 2016 18:20:51 -0800 Subject: [PATCH 218/453] Add documentation to mock types --- multipart/src/mock.rs | 38 +++++++++++++++++++++++++++++++------- 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/multipart/src/mock.rs b/multipart/src/mock.rs index 21e201b58..67011209a 100644 --- a/multipart/src/mock.rs +++ b/multipart/src/mock.rs @@ -4,12 +4,15 @@ // http://apache.org/licenses/LICENSE-2.0> or the MIT license , at your option. This file may not be // copied, modified, or distributed except according to those terms. -//! +//! Mocked types for client-side and server-side APIs. use std::io::{self, Read, Write}; use std::fmt; use rand::{self, Rng, ThreadRng}; +/// A mock implementation of `client::HttpRequest` which can spawn an `HttpBuffer`. +/// +/// `client::HttpRequest` impl requires the `client` feature. #[derive(Default, Debug)] pub struct ClientRequest { boundary: Option, @@ -30,25 +33,34 @@ impl ::client::HttpRequest for ClientRequest { /// ##Panics /// If `apply_headers()` was not called. fn open_stream(self) -> Result { - debug!("MockClientRequest::open_stream called! {:?}", self); - let boundary = self.boundary.expect("HttpRequest::set_headers() was not called!"); + debug!("ClientRequest::open_stream called! {:?}", self); + let boundary = self.boundary.expect("ClientRequest::set_headers() was not called!"); Ok(HttpBuffer::new_empty(boundary, self.content_len)) } } + +/// A writable buffer which stores the boundary and content-length, if provided. +/// +/// Implements `client::HttpStream` if the `client` feature is enabled. pub struct HttpBuffer { + /// The buffer containing the raw bytes. pub buf: Vec, + /// The multipart boundary. pub boundary: String, + /// The value of the content-length header, if set. pub content_len: Option, rng: ThreadRng, } impl HttpBuffer { + /// Create an empty buffer with the given boundary and optional content-length. pub fn new_empty(boundary: String, content_len: Option) -> HttpBuffer { Self::with_buf(Vec::new(), boundary, content_len) } + /// Wrap the given buffer with the given boundary and optional content-length. pub fn with_buf(buf: Vec, boundary: String, content_len: Option) -> Self { HttpBuffer { buf: buf, @@ -58,6 +70,7 @@ impl HttpBuffer { } } + /// Get a `ServerRequest` wrapping the data in this buffer. pub fn for_server(&self) -> ServerRequest { ServerRequest { data: &self.buf, @@ -69,16 +82,18 @@ impl HttpBuffer { } impl Write for HttpBuffer { - fn write(&mut self, out: &[u8]) -> io::Result { - if out.len() == 0 { + /// To simulate a network connection, this will copy a random number of bytes + /// from `buf` to the buffer. + fn write(&mut self, buf: &[u8]) -> io::Result { + if buf.len() == 0 { debug!("HttpBuffer::write() was passed a zero-sized buffer."); return Ok(0); } // Simulate the randomness of a network connection by not always reading everything - let len = self.rng.gen_range(1, out.len() + 1); + let len = self.rng.gen_range(1, buf.len() + 1); - self.buf.write(&out[..len]) + self.buf.write(&buf[..len]) } fn flush(&mut self) -> io::Result<()> { @@ -92,6 +107,7 @@ impl ::client::HttpStream for HttpBuffer { type Response = HttpBuffer; type Error = io::Error; + /// Returns `Ok(self)`. fn finish(self) -> Result { Ok(self) } } @@ -105,14 +121,22 @@ impl fmt::Debug for HttpBuffer { } } +/// A mock implementation of `server::HttpRequest` that can be read. +/// +/// Implements `server::HttpRequest` if the `server` feature is enabled. pub struct ServerRequest<'a> { + /// Slice of the source `HttpBuffer::buf` pub data: &'a [u8], + /// The multipart boundary. pub boundary: &'a str, + /// The value of the content-length header, if set. pub content_len: Option, rng: ThreadRng, } impl<'a> Read for ServerRequest<'a> { + /// To simulate a network connection, this will copy a random number of bytes + /// from the buffer to `out`. fn read(&mut self, out: &mut [u8]) -> io::Result { if out.len() == 0 { debug!("ServerRequest::read() was passed a zero-sized buffer."); From e4441e41ce0cbc5735e759184dc6345ede7ab07c Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Thu, 15 Dec 2016 18:21:04 -0800 Subject: [PATCH 219/453] Bump version, make default features "all" --- multipart/Cargo.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index fe9c51396..2f43ca10a 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "multipart" -version = "0.9.0-alpha.6" +version = "0.9.0-alpha.7" authors = ["Austin Bonander "] @@ -45,9 +45,9 @@ env_logger = "0.3" [features] client = [] -default = ["hyper", "server", "client"] +default = ["all"] server = ["buf_redux", "httparse", "safemem", "twoway"] mock = [] nickel_ = ["nickel", "hyper"] nightly = [] -all = ["iron", "nickel_", "tiny_http"] \ No newline at end of file +all = ["client", "server", "hyper", "iron", "nickel_", "tiny_http", "mock"] \ No newline at end of file From 64fc3ae7280437065de5868406d246c1fe6323ff Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Thu, 15 Dec 2016 18:44:13 -0800 Subject: [PATCH 220/453] Remove mentions of default features since they're all default --- multipart/Cargo.toml | 2 +- multipart/src/client/mod.rs | 3 +-- multipart/src/lib.rs | 22 ++++++++++++---------- multipart/src/server/mod.rs | 3 +-- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 2f43ca10a..d50c4526d 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "multipart" -version = "0.9.0-alpha.7" +version = "0.9.0-alpha.8" authors = ["Austin Bonander "] diff --git a/multipart/src/client/mod.rs b/multipart/src/client/mod.rs index 1dfcf06a0..5960af399 100644 --- a/multipart/src/client/mod.rs +++ b/multipart/src/client/mod.rs @@ -4,8 +4,7 @@ // http://apache.org/licenses/LICENSE-2.0> or the MIT license , at your option. This file may not be // copied, modified, or distributed except according to those terms. -//! The client-side abstraction for multipart requests. Enabled with the `client` feature (on by -//! default). +//! The client-side abstraction for multipart requests. Enabled with the `client` feature. //! //! Use this when sending POST requests with files to a server. use mime::Mime; diff --git a/multipart/src/lib.rs b/multipart/src/lib.rs index 5eb4c7578..30ecdf7d6 100644 --- a/multipart/src/lib.rs +++ b/multipart/src/lib.rs @@ -6,24 +6,26 @@ // copied, modified, or distributed except according to those terms. //! Client- and server-side abstractions for HTTP `multipart/form-data` requests. //! -//! Features: -//! -//! * `client` (default): Enable the client-side abstractions for multipart requests. If the -//! `hyper` feature is also set, enables integration with the Hyper HTTP client API. +//! ###Features: +//! This documentation is built with all features enabled. //! -//! * `server` (default): Enable the server-side abstractions for multipart requests. If the -//! `hyper` feature is also set, enables integration with the Hyper HTTP server API. +//! * `client`: The client-side abstractions for generating multipart requests. //! -//! * `hyper` (default): Enable integration with the [Hyper](https://github.com/hyperium/hyper) HTTP library +//! * `server`: The server-side abstractions for parsing multipart requests. +//! +//! * `mock`: Provides mock implementations of core `client` and `server` traits for debugging +//! or non-standard use. +//! +//! * `hyper`: Integration with the [Hyper](https://github.com/hyperium/hyper) HTTP library //! for client and/or server depending on which other feature flags are set. //! -//! * `iron`: Enable integration with the [Iron](http://ironframework.io) web application +//! * `iron`: Integration with the [Iron](http://ironframework.io) web application //! framework. See the [`server::iron`](server/iron/index.html) module for more information. //! -//! * `tiny_http`: Enable integration with the [`tiny_http`](https://github.com/frewsxcv/tiny-http) +//! * `tiny_http`: Integration with the [`tiny_http`](https://github.com/frewsxcv/tiny-http) //! crate. See the [`server::tiny_http`](server/tiny_http/index.html) module for more information. //! -//! * `nickel_`: Enable integration with the [Nickel](http://nickel.rs) web application framework. +//! * `nickel_`: Integration with the [Nickel](http://nickel.rs) web application framework. //! See the [`server::nickel`](server/nickel/index.html) module for more information. Enables the `hyper` //! feature. #![warn(missing_docs)] diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index 3470bff38..22ee34e42 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -4,8 +4,7 @@ // http://apache.org/licenses/LICENSE-2.0> or the MIT license , at your option. This file may not be // copied, modified, or distributed except according to those terms. -//! The server-side abstraction for multipart requests. Enabled with the `server` feature (on by -//! default). +//! The server-side abstraction for multipart requests. Enabled with the `server` feature. //! //! Use this when you are implementing an HTTP server and want to //! to accept, parse, and serve HTTP `multipart/form-data` requests (file uploads). From a692f8c341aba6af6c97c41745fbbb240901cbe9 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Fri, 16 Dec 2016 17:43:19 -0800 Subject: [PATCH 221/453] Don't require default features of Hyper --- multipart/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index d50c4526d..0c61c773b 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "multipart" -version = "0.9.0-alpha.8" +version = "0.9.0-alpha.9" authors = ["Austin Bonander "] @@ -28,7 +28,7 @@ safemem = { version = "0.2", optional = true } twoway = { version = "0.1", optional = true } # Optional Integrations -hyper = { version = "0.9", optional = true } +hyper = { version = "0.9", optional = true, default-features = false } iron = { version = "0.4", optional = true } # NOTE: use `nickel_` feature, as `hyper` feature is required. From 467e7bd429cab279c70950d91ccab53333fb7b87 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Fri, 16 Dec 2016 17:49:12 -0800 Subject: [PATCH 222/453] Add support for twoway's `pcmp` feature --- multipart/Cargo.toml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 0c61c773b..894ab3471 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -24,6 +24,7 @@ tempdir = ">=0.3.4" #Server Dependencies buf_redux = { version = "0.6", optional = true } +httparse = { version = "1.2", optional = true } safemem = { version = "0.2", optional = true } twoway = { version = "0.1", optional = true } @@ -36,10 +37,6 @@ nickel = { optional = true, version = "0.9" } tiny_http = { version = "0.5", optional = true } -[dependencies.httparse] -version = "1.2" -optional = true - [dev-dependencies] env_logger = "0.3" @@ -50,4 +47,7 @@ server = ["buf_redux", "httparse", "safemem", "twoway"] mock = [] nickel_ = ["nickel", "hyper"] nightly = [] +# Use this to enable SSE4.2 instructions in boundary finding +# TODO: Benchmark this +sse4 = ["nightly", "twoway/pcmp"] all = ["client", "server", "hyper", "iron", "nickel_", "tiny_http", "mock"] \ No newline at end of file From 00d2141acb51fd3fac4b9db850dd84bfc30edab9 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Wed, 18 Jan 2017 14:00:11 -0800 Subject: [PATCH 223/453] Bump to release version --- multipart/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 894ab3471..1c443c7f6 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "multipart" -version = "0.9.0-alpha.9" +version = "0.9.0" authors = ["Austin Bonander "] From 4adabe1df2d9c46afbeb8a54e47b2c63e6f53390 Mon Sep 17 00:00:00 2001 From: Sebastian Blei Date: Tue, 7 Feb 2017 14:34:32 +0100 Subject: [PATCH 224/453] Fixed link-typos in server/mod.rs Due to the case-sensitive file representation on docs.rs For sure, this one does not lack on windows systems -- but the documentation is mostly server on posix systems -- as you know and already said at #61 --- multipart/src/server/mod.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index 22ee34e42..cdf71867e 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -133,7 +133,7 @@ impl Multipart { /// directory under the OS temporary directory. /// /// If there is an error in reading the request, returns the partial result along with the - /// error. See [`SaveResult`](enum.saveresult.html) for more information. + /// error. See [`SaveResult`](enum.SaveResult.html) for more information. pub fn save_all(&mut self) -> SaveResult { let mut entries = match Entries::new_tempdir() { Ok(entries) => entries, @@ -150,7 +150,7 @@ impl Multipart { /// directory under `dir`. /// /// If there is an error in reading the request, returns the partial result along with the - /// error. See [`SaveResult`](enum.saveresult.html) for more information. + /// error. See [`SaveResult`](enum.SaveResult.html) for more information. pub fn save_all_under>(&mut self, dir: P) -> SaveResult { let mut entries = match Entries::new_tempdir_in(dir) { Ok(entries) => entries, @@ -169,7 +169,7 @@ impl Multipart { /// Files larger than `limit` will be truncated to `limit`. /// /// If there is an error in reading the request, returns the partial result along with the - /// error. See [`SaveResult`](enum.saveresult.html) for more information. + /// error. See [`SaveResult`](enum.SaveResult.html) for more information. pub fn save_all_limited(&mut self, limit: u64) -> SaveResult { let mut entries = match Entries::new_tempdir() { Ok(entries) => entries, @@ -188,7 +188,7 @@ impl Multipart { /// Files larger than `limit` will be truncated to `limit`. /// /// If there is an error in reading the request, returns the partial result along with the - /// error. See [`SaveResult`](enum.saveresult.html) for more information. + /// error. See [`SaveResult`](enum.SaveResult.html) for more information. pub fn save_all_under_limited>(&mut self, dir: P, limit: u64) -> SaveResult { let mut entries = match Entries::new_tempdir_in(dir) { Ok(entries) => entries, @@ -245,7 +245,7 @@ impl Borrow for Multipart { } } -/// The result of [`Multipart::save_all()`](struct.multipart.html#method.save_all). +/// The result of [`Multipart::save_all()`](struct.Multipart.html#method.save_all). #[derive(Debug)] pub enum SaveResult { /// The operation was a total success. Contained are all entries of the request. @@ -418,4 +418,4 @@ impl AsRef for SaveDir { fn as_ref(&self) -> &Path { self.as_path() } -} \ No newline at end of file +} From 276a534f01c85f2bd3e7cc22bded9f2c6c687b02 Mon Sep 17 00:00:00 2001 From: Nipunn Koorapati Date: Tue, 14 Feb 2017 21:26:58 -0800 Subject: [PATCH 225/453] Bump the version of iron --- multipart/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 1c443c7f6..6947d4b28 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -30,7 +30,7 @@ twoway = { version = "0.1", optional = true } # Optional Integrations hyper = { version = "0.9", optional = true, default-features = false } -iron = { version = "0.4", optional = true } +iron = { version = "0.5", optional = true } # NOTE: use `nickel_` feature, as `hyper` feature is required. nickel = { optional = true, version = "0.9" } @@ -50,4 +50,4 @@ nightly = [] # Use this to enable SSE4.2 instructions in boundary finding # TODO: Benchmark this sse4 = ["nightly", "twoway/pcmp"] -all = ["client", "server", "hyper", "iron", "nickel_", "tiny_http", "mock"] \ No newline at end of file +all = ["client", "server", "hyper", "iron", "nickel_", "tiny_http", "mock"] From 89f971c4090ddd04bb2d597cc4926d64abf8b728 Mon Sep 17 00:00:00 2001 From: Jeffrey Seyfried Date: Sat, 18 Feb 2017 19:46:41 +0000 Subject: [PATCH 226/453] Remove unused macro def. --- multipart/src/server/field.rs | 9 --------- 1 file changed, 9 deletions(-) diff --git a/multipart/src/server/field.rs b/multipart/src/server/field.rs index 2bed6fc9c..af953168f 100644 --- a/multipart/src/server/field.rs +++ b/multipart/src/server/field.rs @@ -34,15 +34,6 @@ macro_rules! try_io( ) ); -macro_rules! assert_log_ret_none ( - ($expr, $else_:expr) => ( - if !$expr { - $else_; - return None; - } - ) -); - const EMPTY_STR_HEADER: StrHeader<'static> = StrHeader { name: "", val: "", From f993a4db468fccfec3b1f49e313bb105745694ee Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Wed, 18 Jan 2017 20:37:07 -0800 Subject: [PATCH 227/453] Prototype owned/borrowed API --- multipart/Cargo.toml | 2 +- multipart/src/local_test.rs | 7 +- multipart/src/server/field.rs | 258 ++++++++++++++++++++++++++-------- multipart/src/server/iron.rs | 3 +- multipart/src/server/mod.rs | 63 ++++++--- 5 files changed, 249 insertions(+), 84 deletions(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 6947d4b28..98f0b87d4 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "multipart" -version = "0.9.0" +version = "0.10.0-alpha.1" authors = ["Austin Bonander "] diff --git a/multipart/src/local_test.rs b/multipart/src/local_test.rs index cf82f47a6..16d669d49 100644 --- a/multipart/src/local_test.rs +++ b/multipart/src/local_test.rs @@ -162,15 +162,15 @@ fn test_server(buf: HttpBuffer, mut fields: TestFields) { assert!( test_text.is_some(), "Got text field that wasn't in original dataset: {:?} : {:?} ", - field.name, text + field.name, text.text ); let test_text = test_text.unwrap(); assert!( - text == test_text, + text.text == test_text, "Unexpected data for field {:?}: Expected {:?}, got {:?}", - field.name, test_text, text + field.name, test_text, text.text ); }, @@ -184,6 +184,7 @@ fn test_server(buf: HttpBuffer, mut fields: TestFields) { field.name, String::from_utf8_lossy(&test_bytes), String::from_utf8_lossy(&bytes) ); }, + _ => unimplemented!(), } } diff --git a/multipart/src/server/field.rs b/multipart/src/server/field.rs index af953168f..dfe67ec79 100644 --- a/multipart/src/server/field.rs +++ b/multipart/src/server/field.rs @@ -9,16 +9,16 @@ use super::httparse::{self, EMPTY_HEADER, Status}; -use super::Multipart; - -use super::boundary::BoundaryReader; +use super::{Multipart, ReadEntryResult}; use mime::{Attr, Mime, Value}; +use std::borrow::BorrowMut; use std::io::{self, Read, BufRead, Write}; use std::fs::{self, File}; +use std::ops::Deref; use std::path::{Path, PathBuf}; -use std::str; +use std::{mem, str}; const RANDOM_FILENAME_LEN: usize = 12; @@ -46,16 +46,16 @@ struct StrHeader<'a> { } /// The headers that (may) appear before a `multipart/form-data` field. -pub struct FieldHeaders { +struct FieldHeaders { /// The `Content-Disposition` header, required. - pub cont_disp: ContentDisp, + cont_disp: ContentDisp, /// The `Content-Type` header, optional. - pub cont_type: Option, + cont_type: Option, } impl FieldHeaders { /// Parse the field headers from the passed `BufRead`, consuming the relevant bytes. - pub fn parse(r: &mut R) -> io::Result> { + fn read_from(r: &mut R) -> io::Result> { const HEADER_LEN: usize = 4; // These are only written once so they don't need to be `mut` or initialized. @@ -92,16 +92,16 @@ impl FieldHeaders { r.consume(consume); - Ok(Self::read_from(headers)) + Ok(Self::parse(headers)) } - fn read_from(headers: &[StrHeader]) -> Option { + fn parse(headers: &[StrHeader]) -> Option { let cont_disp = try_opt!( - ContentDisp::read_from(headers), + ContentDisp::parse(headers), debug!("Failed to read Content-Disposition") ); - let cont_type = ContentType::read_from(headers); + let cont_type = ContentType::parse(headers); Some(FieldHeaders { cont_disp: cont_disp, @@ -111,15 +111,15 @@ impl FieldHeaders { } /// The `Content-Disposition` header. -pub struct ContentDisp { +struct ContentDisp { /// The name of the `multipart/form-data` field. - pub field_name: String, + field_name: String, /// The optional filename for this field. - pub filename: Option, + filename: Option, } impl ContentDisp { - fn read_from(headers: &[StrHeader]) -> Option { + fn parse(headers: &[StrHeader]) -> Option { if headers.is_empty() { return None; } @@ -165,15 +165,15 @@ impl ContentDisp { } /// The `Content-Type` header. -pub struct ContentType { +struct ContentType { /// The MIME type of the `multipart` field. /// /// May contain a sub-boundary parameter. - pub val: Mime, + val: Mime, } impl ContentType { - fn read_from(headers: &[StrHeader]) -> Option { + fn parse(headers: &[StrHeader]) -> Option { const CONTENT_TYPE: &'static str = "Content-Type"; let header = try_opt!( @@ -196,18 +196,56 @@ impl ContentType { /// A field in a multipart request. May be either text or a binary stream (file). #[derive(Debug)] -pub struct MultipartField<'a, B: 'a> { +pub struct MultipartField { /// The field's name from the form pub name: String, /// The data of the field. Can be text or binary. - pub data: MultipartData<'a, B>, + pub data: MultipartData, } -pub fn read_field(multipart: &mut Multipart) -> io::Result>> { - let field_headers = match multipart.read_field_headers() { - Ok(Some(headers)) => headers, - Ok(None) => return Ok(None), - Err(err) => return Err(err) +impl MultipartField { + /// Read the next entry in the request. + pub fn next_entry(self) -> ReadEntryResult { + self.data.into_inner().read_field() + } + + /// Update `self` as the next entry. + /// + /// Returns `Ok(Some(self))` if another entry was read, `Ok(None)` if the end of the body was + /// reached, and `Err(e)` for any errors that occur. + pub fn next_entry_inplace(&mut self) -> io::Result> where for<'a> &'a mut M: ReadField { + let entry = match try!(self.read_entry()) { + Some(pair) => pair, + None => return Ok(None), + }; + + let multipart = mem::replace(&mut self.data, MultipartData::_Swapping).into_inner(); + + *self = entry.set_inner(multipart); + + Ok(Some(self)) + } + + fn read_entry(&mut self) -> io::Result>> where for<'a> &'a mut M: ReadField { + self.data.inner_mut().read_field() + .map_err(|e| e.error) + .map(|entry| entry.map(|entry| entry.set_inner(()))) + } +} + +impl MultipartField { + fn set_inner(self, new_inner: M_) -> MultipartField { + MultipartField { + name: self.name, + data: self.data.set_inner(new_inner).1 + } + } +} + +pub fn read_field>>(mut multipart: M) -> ReadEntryResult { + let field_headers = match try_read_entry!(multipart; FieldHeaders::read_from(&mut multipart.borrow_mut().reader)) { + Some(headers) => headers, + None => return Ok(None), }; let data = match field_headers.cont_type { @@ -216,13 +254,16 @@ pub fn read_field(multipart: &mut Multipart) -> io::Result { - let text = try!(multipart.read_to_string()); - MultipartData::Text(&text) + let text = try_read_entry!(multipart; multipart.borrow_mut().read_to_string()); + MultipartData::Text(MultipartText { + text: text, + multipart: multipart, + }) }, }; @@ -236,27 +277,64 @@ pub fn read_field(multipart: &mut Multipart) -> io::Result { +pub enum MultipartData { /// The field's payload is a text string. - Text(&'a str), + Text(MultipartText), /// The field's payload is a binary stream (file). - File(MultipartFile<'a, B>), + File(MultipartFile), // TODO: Support multiple files per field (nested boundaries) // MultiFiles(Vec), + #[doc(hidden)] + _Swapping, } -impl<'a, B> MultipartData<'a, B> { +impl MultipartData { + /// Return the inner `Multipart`. + pub fn into_inner(self) -> M { + use self::MultipartData::*; + + match self { + Text(text) => text.multipart, + File(file) => file.multipart, + _Swapping => unreachable!("MultipartData::_Swapping was left in-place somehow"), + } + } + + fn inner_mut(&mut self) -> &mut M { + use self::MultipartData::*; + + match *self { + Text(ref mut text) => &mut text.multipart, + File(ref mut file) => &mut file.multipart, + _Swapping => unreachable!("MultipartData::_Swapping was left in-place somehow"), + } + } + + fn set_inner(self, new_inner: M_) -> (M, MultipartData) { + use self::MultipartData::*; + + match self { + Text(text) => (text.multipart, Text(MultipartText { text: text.text, multipart: new_inner })), + File(file) => (file.multipart, File(MultipartFile { + filename: file.filename, + content_type: file.content_type, + multipart: new_inner + })), + _Swapping => unreachable!("MultipartData::_Swapping was left in-place somehow"), + } + } + /// Borrow this payload as a text field, if possible. pub fn as_text(&self) -> Option<&str> { match *self { - MultipartData::Text(ref s) => Some(s), + MultipartData::Text(ref text) => Some(&text.text), _ => None, } } /// Borrow this payload as a file field, if possible. /// Mutably borrows so the contents can be read. - pub fn as_file(&mut self) -> Option<&mut MultipartFile<'a, B>> { + pub fn as_file(&mut self) -> Option<&mut MultipartFile> { match *self { MultipartData::File(ref mut file) => Some(file), _ => None, @@ -264,6 +342,29 @@ impl<'a, B> MultipartData<'a, B> { } } +/// A representation of a text field in a `multipart/form-data` body. +#[derive(Debug)] +pub struct MultipartText { + /// The text of this field. + pub text: String, + /// The `Multipart` this field was read from. + pub multipart: M, +} + +impl Deref for MultipartText { + type Target = str; + + fn deref(&self) -> &Self::Target { + &self.text + } +} + +impl Into for MultipartText { + fn into(self) -> String { + self.text + } +} + /// A representation of a file in HTTP `multipart/form-data`. /// /// Note that the file is not yet saved to the local filesystem; @@ -273,30 +374,49 @@ impl<'a, B> MultipartData<'a, B> { /// You can read it to EOF, or use one of the `save_*()` methods here /// to save it to disk. #[derive(Debug)] -pub struct MultipartFile<'a, B: 'a> { +pub struct MultipartFile { filename: Option, content_type: Mime, - stream: &'a mut BoundaryReader, + /// The `Multipart` this field was read from. + pub multipart: M, } -impl<'a, B: Read> MultipartFile<'a, B> { +impl MultipartFile { fn from_stream(filename: Option, content_type: Mime, - stream: &'a mut BoundaryReader) -> MultipartFile<'a, B> { + multipart: M) -> MultipartFile { MultipartFile { filename: filename, content_type: content_type, - stream: stream, + multipart: multipart, } } + /// Get the filename of this entry, if supplied. + /// + /// ##Warning + /// You should treat this value as untrustworthy because it is an arbitrary string provided by + /// the client. You should *not* blindly append it to a directory path and save the file there, + /// as such behavior could easily be exploited by a malicious client. + pub fn filename(&self) -> Option<&str> { + self.filename.as_ref().map(String::as_ref) + } + + /// Get the MIME type (`Content-Type` value) of this file, if supplied by the client, + /// or `"applicaton/octet-stream"` otherwise. + pub fn content_type(&self) -> &Mime { + &self.content_type + } +} + +impl MultipartFile where MultipartFile: Read { /// Save this file to the given output stream. /// /// If successful, returns the number of bytes written. /// /// Retries when `io::Error::kind() == io::ErrorKind::Interrupted`. pub fn save_to(&mut self, mut out: W) -> io::Result { - retry_on_interrupt(|| io::copy(self.stream, &mut out)) + retry_on_interrupt(|| io::copy(self, &mut out)) } /// Save this file to the given output stream, **truncated** to `limit` @@ -306,7 +426,7 @@ impl<'a, B: Read> MultipartFile<'a, B> { /// /// Retries when `io::Error::kind() == io::ErrorKind::Interrupted`. pub fn save_to_limited(&mut self, mut out: W, limit: u64) -> io::Result { - retry_on_interrupt(|| io::copy(&mut self.stream.take(limit), &mut out)) + retry_on_interrupt(|| io::copy(&mut self.by_ref().take(limit), &mut out)) } /// Save this file to `path`. @@ -372,37 +492,37 @@ impl<'a, B: Read> MultipartFile<'a, B> { let path = dir.as_ref().join(::random_alphanumeric(RANDOM_FILENAME_LEN)); self.save_as_limited(path, limit) } +} - /// Get the filename of this entry, if supplied. - /// - /// ##Warning - /// You should treat this value as untrustworthy because it is an arbitrary string provided by - /// the client. You should *not* blindly append it to a directory path and save the file there, - /// as such behavior could easily be exploited by a malicious client. - pub fn filename(&self) -> Option<&str> { - self.filename.as_ref().map(String::as_ref) +impl Read for MultipartFile> { + fn read(&mut self, buf: &mut [u8]) -> io::Result{ + self.multipart.reader.read(buf) } +} - /// Get the MIME type (`Content-Type` value) of this file, if supplied by the client, - /// or `"applicaton/octet-stream"` otherwise. - pub fn content_type(&self) -> &Mime { - &self.content_type +impl BufRead for MultipartFile> { + fn fill_buf(&mut self) -> io::Result<&[u8]> { + self.multipart.reader.fill_buf() + } + + fn consume(&mut self, amt: usize) { + self.multipart.reader.consume(amt) } } -impl<'a, B: Read> Read for MultipartFile<'a, B> { +impl<'a, R: Read + 'a> Read for MultipartFile<&'a mut Multipart> { fn read(&mut self, buf: &mut [u8]) -> io::Result{ - self.stream.read(buf) + self.multipart.reader.read(buf) } } -impl<'a, B: Read> BufRead for MultipartFile<'a, B> { +impl<'a, R: Read + 'a> BufRead for MultipartFile<&'a mut Multipart> { fn fill_buf(&mut self) -> io::Result<&[u8]> { - self.stream.fill_buf() + self.multipart.reader.fill_buf() } fn consume(&mut self, amt: usize) { - self.stream.consume(amt) + self.multipart.borrow_mut().reader.consume(amt) } } @@ -424,8 +544,6 @@ pub struct SavedFile { pub size: u64, } - - fn read_content_type(cont_type: &str) -> Mime { cont_type.parse().ok().unwrap_or_else(::mime_guess::octet_stream) } @@ -474,4 +592,20 @@ fn create_full_path(path: &Path) -> io::Result { } File::create(&path) -} \ No newline at end of file +} + +pub trait ReadField: Sized { + fn read_field(self) -> ReadEntryResult; +} + +impl ReadField for Multipart { + fn read_field(self) -> ReadEntryResult { + read_field(self) + } +} + +impl<'a, R: Read + 'a> ReadField for &'a mut Multipart { + fn read_field(self) -> ReadEntryResult { + read_field(self) + } +} diff --git a/multipart/src/server/iron.rs b/multipart/src/server/iron.rs index fe1c4d718..315ded22b 100644 --- a/multipart/src/server/iron.rs +++ b/multipart/src/server/iron.rs @@ -171,8 +171,9 @@ impl Intercept { file_count += 1; }, MultipartData::Text(text) => { - entries.fields.insert(field.name, text.into()); + entries.fields.insert(field.name, text.text); }, + MultipartData::_Swapping => unreachable!("MultipartData::_Swapping was left in-place somehow"), } } diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index cdf71867e..b269690a9 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -26,7 +26,6 @@ use std::path::{Path, PathBuf}; use std::{io, mem}; use self::boundary::BoundaryReader; -use self::field::FieldHeaders; pub use self::field::{MultipartField, MultipartFile, MultipartData, SavedFile}; @@ -48,6 +47,19 @@ macro_rules! try_opt ( ) ); +macro_rules! try_read_entry { + ($self_:expr; $try:expr) => ( + match $try { + Ok(res) => res, + Err(err) => return Err(::server::ReadEntryError { + multipart: $self_, + error: err, + __private: (), + }) + } + ) +} + mod boundary; mod field; @@ -67,8 +79,7 @@ pub mod tiny_http; /// /// Implements `Borrow` to allow access to the request body, if desired. pub struct Multipart { - source: BoundaryReader, - line_buf: String, + reader: BoundaryReader, } impl Multipart<()> { @@ -90,8 +101,7 @@ impl Multipart { /// Construct a new `Multipart` with the given body reader and boundary. pub fn with_body>(body: B, boundary: Bnd) -> Self { Multipart { - source: BoundaryReader::from_reader(body, boundary.into()), - line_buf: String::new(), + reader: BoundaryReader::from_reader(body, boundary.into()), } } @@ -101,16 +111,22 @@ impl Multipart { /// ##Warning: Risk of Data Loss /// If the previously returned entry had contents of type `MultipartField::File`, /// calling this again will discard any unread contents of that entry. - pub fn read_entry(&mut self) -> io::Result>> { + pub fn read_entry(&mut self) -> io::Result>> { if try!(self.consume_boundary()) { return Ok(None); } - self::field::read_field(self) + self::field::read_field(self).map_err(|e| e.error) } - fn read_field_headers(&mut self) -> io::Result> { - FieldHeaders::parse(&mut self.source) + /// Read the next entry from this multipart request, returning a struct with the field's name and + /// data. See `MultipartField` for more info. + pub fn into_entry(mut self) -> ReadEntryResult { + if try_read_entry!(self; self.consume_boundary()) { + return Ok(None); + } + + self::field::read_field(self) } /// Call `f` for each entry in the multipart request. @@ -119,7 +135,7 @@ impl Multipart { /// from `next()` borrows the iterator for a bound lifetime). /// /// Returns `Ok(())` when all fields have been read, or the first error. - pub fn foreach_entry(&mut self, mut foreach: F) -> io::Result<()> where F: FnMut(MultipartField) { + pub fn foreach_entry(&mut self, mut foreach: F) -> io::Result<()> where F: FnMut(MultipartField<&mut Self>) { loop { match self.read_entry() { Ok(Some(field)) => foreach(field), @@ -214,19 +230,20 @@ impl Multipart { entries.files.insert(field.name, file); }, MultipartData::Text(text) => { - entries.fields.insert(field.name, text.into()); + entries.fields.insert(field.name, text.text); }, + MultipartData::_Swapping => unreachable!("MultipartData::_Swapping was left in-place somehow"), } } Ok(()) } - fn read_to_string(&mut self) -> io::Result<&str> { - self.line_buf.clear(); + fn read_to_string(&mut self) -> io::Result { + let mut buf = String::new(); - match self.source.read_to_string(&mut self.line_buf) { - Ok(read) => Ok(&self.line_buf[..read]), + match self.reader.read_to_string(&mut buf) { + Ok(_) => Ok(buf), Err(err) => Err(err), } } @@ -235,13 +252,13 @@ impl Multipart { /// Returns `true` if the last boundary was read, `false` otherwise. fn consume_boundary(&mut self) -> io::Result { debug!("Consume boundary!"); - self.source.consume_boundary() + self.reader.consume_boundary() } } impl Borrow for Multipart { fn borrow(&self) -> &B { - self.source.borrow() + self.reader.borrow() } } @@ -419,3 +436,15 @@ impl AsRef for SaveDir { self.as_path() } } + +/// Result type returned by `Multipart::read_entry()` and `Multipart::into_entry()`. +pub type ReadEntryResult = Result>, ReadEntryError>; + +/// Error type returned by `Multipart::read_entry()` and `Multipart::into_entry()`. +pub struct ReadEntryError { + /// The `Multipart` that caused the error. + pub multipart: M, + /// The actual error. + pub error: io::Error, + __private: (), +} From c5fff1c40fff818ebf8acbd4139bcdbd07f2fca2 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Fri, 20 Jan 2017 01:43:05 -0800 Subject: [PATCH 228/453] Prototype nested boundaries --- multipart/src/server/boundary.rs | 12 +- multipart/src/server/field.rs | 421 +++++++++++++++++++------------ multipart/src/server/iron.rs | 2 +- multipart/src/server/mod.rs | 29 +-- 4 files changed, 283 insertions(+), 181 deletions(-) diff --git a/multipart/src/server/boundary.rs b/multipart/src/server/boundary.rs index d9b4de56c..95c66b044 100644 --- a/multipart/src/server/boundary.rs +++ b/multipart/src/server/boundary.rs @@ -18,6 +18,8 @@ use std::borrow::Borrow; use std::io; use std::io::prelude::*; +use std::mem; + /// A struct implementing `Read` and `BufRead` that will yield bytes until it sees a given sequence. #[derive(Debug)] pub struct BoundaryReader { @@ -169,8 +171,14 @@ impl BoundaryReader where R: Read { // Keeping this around to support nested boundaries later. #[allow(unused)] #[doc(hidden)] - pub fn set_boundary>>(&mut self, boundary: B) { - self.boundary = boundary.into(); + pub fn swap_boundary>>(&mut self, boundary: B) -> Vec { + let mut boundary = boundary.into(); + + if boundary.get(..2) != Some(b"--") { + safemem::prepend(b"--", &mut boundary); + } + + mem::replace(&mut self.boundary, boundary) } } diff --git a/multipart/src/server/field.rs b/multipart/src/server/field.rs index dfe67ec79..04c911444 100644 --- a/multipart/src/server/field.rs +++ b/multipart/src/server/field.rs @@ -9,13 +9,13 @@ use super::httparse::{self, EMPTY_HEADER, Status}; -use super::{Multipart, ReadEntryResult}; +use super::Multipart; +use self::ReadEntryResult::*; -use mime::{Attr, Mime, Value}; +use mime::{Attr, TopLevel, Mime, Value}; -use std::borrow::BorrowMut; use std::io::{self, Read, BufRead, Write}; -use std::fs::{self, File}; +use std::fs; use std::ops::Deref; use std::path::{Path, PathBuf}; use std::{mem, str}; @@ -46,11 +46,11 @@ struct StrHeader<'a> { } /// The headers that (may) appear before a `multipart/form-data` field. -struct FieldHeaders { +pub struct FieldHeaders { /// The `Content-Disposition` header, required. cont_disp: ContentDisp, /// The `Content-Type` header, optional. - cont_type: Option, + cont_type: Option, } impl FieldHeaders { @@ -101,17 +101,21 @@ impl FieldHeaders { debug!("Failed to read Content-Disposition") ); - let cont_type = ContentType::parse(headers); + let cont_type = parse_cont_type(headers); Some(FieldHeaders { cont_disp: cont_disp, cont_type: cont_type, }) } + + fn name(&self) -> &str { + &self.cont_disp.field_name + } } /// The `Content-Disposition` header. -struct ContentDisp { +pub struct ContentDisp { /// The name of the `multipart/form-data` field. field_name: String, /// The optional filename for this field. @@ -164,163 +168,78 @@ impl ContentDisp { } } -/// The `Content-Type` header. -struct ContentType { - /// The MIME type of the `multipart` field. - /// - /// May contain a sub-boundary parameter. - val: Mime, -} - -impl ContentType { - fn parse(headers: &[StrHeader]) -> Option { - const CONTENT_TYPE: &'static str = "Content-Type"; - - let header = try_opt!( - find_header(headers, CONTENT_TYPE), - debug!("Content-Type header not found for field.") - ); +fn parse_cont_type(headers: &[StrHeader]) -> Option { + const CONTENT_TYPE: &'static str = "Content-Type"; - // Boundary parameter will be parsed into the `Mime` - debug!("Found Content-Type: {:?}", header.val); - let content_type = read_content_type(header.val.trim()); - Some(ContentType { val: content_type }) - } + let header = try_opt!( + find_header(headers, CONTENT_TYPE), + debug!("Content-Type header not found for field.") + ); - /// Get the optional boundary parameter for this `Content-Type`. - #[allow(dead_code)] - pub fn boundary(&self) -> Option<&str> { - self.val.get_param(Attr::Boundary).map(Value::as_str) - } + // Boundary parameter will be parsed into the `Mime` + debug!("Found Content-Type: {:?}", header.val); + let content_type = read_content_type(header.val.trim()); + Some(content_type) } /// A field in a multipart request. May be either text or a binary stream (file). #[derive(Debug)] -pub struct MultipartField { +pub struct MultipartField { /// The field's name from the form pub name: String, /// The data of the field. Can be text or binary. pub data: MultipartData, } -impl MultipartField { +impl MultipartField { /// Read the next entry in the request. pub fn next_entry(self) -> ReadEntryResult { - self.data.into_inner().read_field() + self.data.into_inner().read_entry() } /// Update `self` as the next entry. /// /// Returns `Ok(Some(self))` if another entry was read, `Ok(None)` if the end of the body was /// reached, and `Err(e)` for any errors that occur. - pub fn next_entry_inplace(&mut self) -> io::Result> where for<'a> &'a mut M: ReadField { - let entry = match try!(self.read_entry()) { - Some(pair) => pair, - None => return Ok(None), - }; - - let multipart = mem::replace(&mut self.data, MultipartData::_Swapping).into_inner(); - - *self = entry.set_inner(multipart); - - Ok(Some(self)) - } - - fn read_entry(&mut self) -> io::Result>> where for<'a> &'a mut M: ReadField { - self.data.inner_mut().read_field() - .map_err(|e| e.error) - .map(|entry| entry.map(|entry| entry.set_inner(()))) - } -} + pub fn next_entry_inplace(&mut self) -> io::Result> where for<'a> &'a mut M: ReadEntry { + let multipart = self.data.take_inner(); -impl MultipartField { - fn set_inner(self, new_inner: M_) -> MultipartField { - MultipartField { - name: self.name, - data: self.data.set_inner(new_inner).1 + match multipart.read_entry() { + Entry(entry) => { + *self = entry; + Ok(Some(self)) + }, + End(multipart) => { + self.data.give_inner(multipart); + Ok(None) + }, + Error(multipart, err) => { + self.data.give_inner(multipart); + Err(err) + } } } } -pub fn read_field>>(mut multipart: M) -> ReadEntryResult { - let field_headers = match try_read_entry!(multipart; FieldHeaders::read_from(&mut multipart.borrow_mut().reader)) { - Some(headers) => headers, - None => return Ok(None), - }; - - let data = match field_headers.cont_type { - Some(content_type) => { - MultipartData::File( - MultipartFile::from_stream( - field_headers.cont_disp.filename, - content_type.val, - multipart, - ) - ) - }, - None => { - let text = try_read_entry!(multipart; multipart.borrow_mut().read_to_string()); - MultipartData::Text(MultipartText { - text: text, - multipart: multipart, - }) - }, - }; - - Ok(Some( - MultipartField { - name: field_headers.cont_disp.field_name, - data: data, - } - )) -} - /// The data of a field in a `multipart/form-data` request. #[derive(Debug)] -pub enum MultipartData { +pub enum MultipartData { /// The field's payload is a text string. Text(MultipartText), /// The field's payload is a binary stream (file). File(MultipartFile), - // TODO: Support multiple files per field (nested boundaries) - // MultiFiles(Vec), - #[doc(hidden)] - _Swapping, + /// The field's payload is a nested multipart body (multiple files). + Nested(NestedMultipart), } -impl MultipartData { - /// Return the inner `Multipart`. - pub fn into_inner(self) -> M { - use self::MultipartData::*; - - match self { - Text(text) => text.multipart, - File(file) => file.multipart, - _Swapping => unreachable!("MultipartData::_Swapping was left in-place somehow"), - } - } - +impl MultipartData { fn inner_mut(&mut self) -> &mut M { use self::MultipartData::*; match *self { - Text(ref mut text) => &mut text.multipart, - File(ref mut file) => &mut file.multipart, - _Swapping => unreachable!("MultipartData::_Swapping was left in-place somehow"), - } - } - - fn set_inner(self, new_inner: M_) -> (M, MultipartData) { - use self::MultipartData::*; - - match self { - Text(text) => (text.multipart, Text(MultipartText { text: text.text, multipart: new_inner })), - File(file) => (file.multipart, File(MultipartFile { - filename: file.filename, - content_type: file.content_type, - multipart: new_inner - })), - _Swapping => unreachable!("MultipartData::_Swapping was left in-place somehow"), + Text(ref mut text) => text.inner_mut(), + File(ref mut file) => file.inner_mut(), + Nested(ref mut nested) => nested.inner_mut(), } } @@ -340,6 +259,39 @@ impl MultipartData { _ => None, } } + + /// Return the inner `Multipart`. + pub fn into_inner(self) -> M { + use self::MultipartData::*; + + match self { + Text(text) => text.into_inner(), + File(file) => file.into_inner(), + Nested(nested) => nested.into_inner(), + } + } + + fn take_inner(&mut self) -> M { + use self::MultipartData::*; + + match *self { + Text(ref mut text) => text.take_inner(), + File(ref mut file) => file.take_inner(), + Nested(ref mut nested) => nested.take_inner(), + } + } + + fn give_inner(&mut self, inner: M) { + use self::MultipartData::*; + + let inner = Some(inner); + + match *self { + Text(ref mut text) => text.inner = inner, + File(ref mut file) => file.inner = inner, + Nested(ref mut nested) => nested.inner = inner, + } + } } /// A representation of a text field in a `multipart/form-data` body. @@ -348,7 +300,7 @@ pub struct MultipartText { /// The text of this field. pub text: String, /// The `Multipart` this field was read from. - pub multipart: M, + inner: Option, } impl Deref for MultipartText { @@ -365,6 +317,20 @@ impl Into for MultipartText { } } +impl MultipartText { + fn inner_mut(&mut self) -> &mut M { + self.inner.as_mut().expect("MultipartText::inner taken!") + } + + fn take_inner(&mut self) -> M { + self.inner.take().expect("MultipartText::inner taken!") + } + + fn into_inner(self) -> M { + self.inner.expect("MultipartText::inner taken!") + } +} + /// A representation of a file in HTTP `multipart/form-data`. /// /// Note that the file is not yet saved to the local filesystem; @@ -378,20 +344,10 @@ pub struct MultipartFile { filename: Option, content_type: Mime, /// The `Multipart` this field was read from. - pub multipart: M, + inner: Option, } impl MultipartFile { - fn from_stream(filename: Option, - content_type: Mime, - multipart: M) -> MultipartFile { - MultipartFile { - filename: filename, - content_type: content_type, - multipart: multipart, - } - } - /// Get the filename of this entry, if supplied. /// /// ##Warning @@ -407,6 +363,18 @@ impl MultipartFile { pub fn content_type(&self) -> &Mime { &self.content_type } + + fn inner_mut(&mut self) -> &mut M { + self.inner.as_mut().expect("MultipartFile::inner taken!") + } + + fn take_inner(&mut self) -> M { + self.inner.take().expect("MultipartFile::inner taken!") + } + + fn into_inner(self) -> M { + self.inner.expect("MultipartFile::inner taken!") + } } impl MultipartFile where MultipartFile: Read { @@ -496,33 +464,33 @@ impl MultipartFile where MultipartFile: Read { impl Read for MultipartFile> { fn read(&mut self, buf: &mut [u8]) -> io::Result{ - self.multipart.reader.read(buf) + self.inner_mut().reader.read(buf) } } impl BufRead for MultipartFile> { fn fill_buf(&mut self) -> io::Result<&[u8]> { - self.multipart.reader.fill_buf() + self.inner_mut().reader.fill_buf() } fn consume(&mut self, amt: usize) { - self.multipart.reader.consume(amt) + self.inner_mut().reader.consume(amt) } } impl<'a, R: Read + 'a> Read for MultipartFile<&'a mut Multipart> { fn read(&mut self, buf: &mut [u8]) -> io::Result{ - self.multipart.reader.read(buf) + self.inner_mut().reader.read(buf) } } impl<'a, R: Read + 'a> BufRead for MultipartFile<&'a mut Multipart> { fn fill_buf(&mut self) -> io::Result<&[u8]> { - self.multipart.reader.fill_buf() + self.inner_mut().reader.fill_buf() } fn consume(&mut self, amt: usize) { - self.multipart.borrow_mut().reader.consume(amt) + self.inner_mut().reader.consume(amt) } } @@ -583,7 +551,7 @@ fn retry_on_interrupt(mut do_fn: F) -> io::Result where F: FnMut() -> i } } -fn create_full_path(path: &Path) -> io::Result { +fn create_full_path(path: &Path) -> io::Result { if let Some(parent) = path.parent() { try!(fs::create_dir_all(parent)); } else { @@ -591,21 +559,162 @@ fn create_full_path(path: &Path) -> io::Result { warn!("Attempting to save file in what looks like a root directory. File path: {:?}", path); } - File::create(&path) + fs::File::create(&path) +} + +#[derive(Debug)] +pub struct NestedMultipart { + outer_boundary: Vec, + inner: Option, +} + +impl NestedMultipart { + pub fn read_entry(&mut self) -> ReadEntryResult<&mut M> where for<'a> &'a mut M: ReadEntry { + self.inner.as_mut().expect("NestedMultipart::inner taken!()").read_entry() + } + + fn inner_mut(&mut self) -> &mut M { + self.inner.as_mut().expect("NestedMultipart::inner taken!") + } + + fn take_inner(&mut self) -> M { + self.inner.take().expect("NestedMultipart::inner taken!()") + } + + fn into_inner(mut self) -> M { + self.restore_boundary(); + self.inner.take().expect("NestedMultipart::inner taken!()") + } + + fn restore_boundary(&mut self) { + let multipart = self.inner.as_mut().expect("NestedMultipart::inner taken!()"); + let outer_boundary = mem::replace(&mut self.outer_boundary, Vec::new()); + multipart.swap_boundary(outer_boundary); + } +} + +impl Drop for NestedMultipart { + fn drop(&mut self) { + if self.inner.is_some() { + self.restore_boundary(); + } + } } -pub trait ReadField: Sized { - fn read_field(self) -> ReadEntryResult; +/// Public trait but not re-exported. +pub trait ReadEntry: Sized { + fn read_headers(&mut self) -> io::Result>; + + fn read_to_string(&mut self) -> io::Result; + + fn swap_boundary>>(&mut self, boundary: B) -> Vec; + + fn read_entry(mut self) -> ReadEntryResult { + let field_headers = match try_read_entry!(self; self.read_headers()) { + Some(headers) => headers, + None => return End(self), + }; + + let data = match field_headers.cont_type { + Some(cont_type) => { + match cont_type.0 { + TopLevel::Multipart if cont_type.1 == "mixed" => { + let outer_boundary = match cont_type.get_param(Attr::Boundary) { + Some(&Value::Ext(ref boundary)) => self.swap_boundary(&**boundary), + _ => { + let msg = format!("Nested multipart boundary was not provided for \ + field {:?}", field_headers.cont_disp.field_name); + return ReadEntryResult::invalid_data(self, msg); + }, + }; + + MultipartData::Nested( + NestedMultipart { + outer_boundary: outer_boundary, + inner: Some(self), + } + ) + }, + _ => { + MultipartData::File( + MultipartFile { + filename: field_headers.cont_disp.filename, + content_type: cont_type, + inner: Some(self) + } + ) + } + } + }, + None => { + let text = try_read_entry!(self; self.read_to_string()); + MultipartData::Text(MultipartText { + text: text, + inner: Some(self), + }) + }, + }; + + Entry( + MultipartField { + name: field_headers.cont_disp.field_name, + data: data, + } + ) + } } -impl ReadField for Multipart { - fn read_field(self) -> ReadEntryResult { - read_field(self) +impl ReadEntry for Multipart { + fn read_headers(&mut self) -> io::Result> { + FieldHeaders::read_from(&mut self.reader) + } + + fn read_to_string(&mut self) -> io::Result { + self.read_to_string() + } + + fn swap_boundary>>(&mut self, boundary: B) -> Vec { + self.reader.swap_boundary(boundary) } } -impl<'a, R: Read + 'a> ReadField for &'a mut Multipart { - fn read_field(self) -> ReadEntryResult { - read_field(self) +impl<'a, M: ReadEntry> ReadEntry for &'a mut M { + fn read_headers(&mut self) -> io::Result> { + self.read_headers() + } + + fn read_to_string(&mut self) -> io::Result { + self.read_to_string() + } + + fn swap_boundary>>(&mut self, boundary: B) -> Vec { + self.swap_boundary(boundary) } } + +/// Result type returned by `Multipart::into_entry()` and `MultipartField::next_entry()`. +pub enum ReadEntryResult { + /// The next entry was found. + Entry(MultipartField), + /// No more entries could be read. + End(M), + /// An error occurred. + Error(M, io::Error), +} + +impl ReadEntryResult { + pub fn into_result(self) -> io::Result>> { + match self { + ReadEntryResult::Entry(entry) => Ok(Some(entry)), + ReadEntryResult::End(_) => Ok(None), + ReadEntryResult::Error(_, err) => Err(err), + } + } + + fn invalid_data(multipart: M, msg: String) -> Self { + ReadEntryResult::Error ( + multipart, + io::Error::new(io::ErrorKind::InvalidData, msg), + ) + } +} \ No newline at end of file diff --git a/multipart/src/server/iron.rs b/multipart/src/server/iron.rs index 315ded22b..7643cff09 100644 --- a/multipart/src/server/iron.rs +++ b/multipart/src/server/iron.rs @@ -173,7 +173,7 @@ impl Intercept { MultipartData::Text(text) => { entries.fields.insert(field.name, text.text); }, - MultipartData::_Swapping => unreachable!("MultipartData::_Swapping was left in-place somehow"), + MultipartData::Nested(nested) => unimplemented!(), } } diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index b269690a9..0e8559811 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -26,8 +26,9 @@ use std::path::{Path, PathBuf}; use std::{io, mem}; use self::boundary::BoundaryReader; +use self::field::ReadEntry; -pub use self::field::{MultipartField, MultipartFile, MultipartData, SavedFile}; +pub use self::field::{MultipartField, MultipartFile, MultipartData, ReadEntryResult, SavedFile}; macro_rules! try_opt ( ($expr:expr) => ( @@ -51,11 +52,7 @@ macro_rules! try_read_entry { ($self_:expr; $try:expr) => ( match $try { Ok(res) => res, - Err(err) => return Err(::server::ReadEntryError { - multipart: $self_, - error: err, - __private: (), - }) + Err(err) => return ::server::ReadEntryResult::Error($self_, err), } ) } @@ -116,17 +113,17 @@ impl Multipart { return Ok(None); } - self::field::read_field(self).map_err(|e| e.error) + ReadEntry::read_entry(self).into_result() } /// Read the next entry from this multipart request, returning a struct with the field's name and /// data. See `MultipartField` for more info. pub fn into_entry(mut self) -> ReadEntryResult { if try_read_entry!(self; self.consume_boundary()) { - return Ok(None); + return ReadEntryResult::End(self); } - self::field::read_field(self) + self.read_entry() } /// Call `f` for each entry in the multipart request. @@ -232,7 +229,7 @@ impl Multipart { MultipartData::Text(text) => { entries.fields.insert(field.name, text.text); }, - MultipartData::_Swapping => unreachable!("MultipartData::_Swapping was left in-place somehow"), + MultipartData::Nested(_) => unimplemented!(), } } @@ -436,15 +433,3 @@ impl AsRef for SaveDir { self.as_path() } } - -/// Result type returned by `Multipart::read_entry()` and `Multipart::into_entry()`. -pub type ReadEntryResult = Result>, ReadEntryError>; - -/// Error type returned by `Multipart::read_entry()` and `Multipart::into_entry()`. -pub struct ReadEntryError { - /// The `Multipart` that caused the error. - pub multipart: M, - /// The actual error. - pub error: io::Error, - __private: (), -} From 3c13bb1fc93b7a2b35efa3034d4bc3b8736f1e69 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Fri, 20 Jan 2017 19:54:13 -0800 Subject: [PATCH 229/453] Remove unused impl details --- multipart/src/server/field.rs | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/multipart/src/server/field.rs b/multipart/src/server/field.rs index 04c911444..7fba811c0 100644 --- a/multipart/src/server/field.rs +++ b/multipart/src/server/field.rs @@ -108,10 +108,6 @@ impl FieldHeaders { cont_type: cont_type, }) } - - fn name(&self) -> &str { - &self.cont_disp.field_name - } } /// The `Content-Disposition` header. @@ -233,16 +229,6 @@ pub enum MultipartData { } impl MultipartData { - fn inner_mut(&mut self) -> &mut M { - use self::MultipartData::*; - - match *self { - Text(ref mut text) => text.inner_mut(), - File(ref mut file) => file.inner_mut(), - Nested(ref mut nested) => nested.inner_mut(), - } - } - /// Borrow this payload as a text field, if possible. pub fn as_text(&self) -> Option<&str> { match *self { From e022e69167f3193618904b254e3454805ae7e5d8 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sat, 4 Feb 2017 06:09:51 -0800 Subject: [PATCH 230/453] Add nested file fields to local_test --- multipart/src/local_test.rs | 124 ++++++++++++++++++++++++++++++++---- 1 file changed, 111 insertions(+), 13 deletions(-) diff --git a/multipart/src/local_test.rs b/multipart/src/local_test.rs index 16d669d49..c5472bf82 100644 --- a/multipart/src/local_test.rs +++ b/multipart/src/local_test.rs @@ -16,7 +16,32 @@ use std::iter; #[derive(Debug)] struct TestFields { texts: HashMap, - files: HashMap>, + files: HashMap, + nested: HashMap>, +} + +#[derive(Debug)] +struct FileEntry { + filename: Option, + data: Vec, +} + +impl FileEntry { + fn gen() -> Self { + let filename = match gen_bool() { + true => Some(gen_string()), + false => None, + }; + + FileEntry { + filename: filename, + data: gen_bytes() + } + } + + fn filename(&self) -> Option<&str> { + self.filename.as_ref().map(|s| &**s) + } } #[test] @@ -56,13 +81,26 @@ fn gen_test_fields() -> TestFields { let texts_count = rng.gen_range(MIN_FIELDS, MAX_FIELDS); let files_count = rng.gen_range(MIN_FIELDS, MAX_FIELDS); + let nested_count = rng.gen_range(MIN_FIELDS, MAX_FIELDS); TestFields { texts: (0..texts_count).map(|_| (gen_string(), gen_string())).collect(), - files: (0..files_count).map(|_| (gen_string(), gen_bytes())).collect(), + files: (0..files_count).map(|_| (gen_string(), FileEntry::gen())).collect(), + nested: (0..nested_count).map(|_| { + let files_count = rng.gen_range(MIN_FIELDS, MAX_FIELDS); + + ( + gen_string(), + (0..files_count).map(|_| FileEntry::gen()).collect() + ) + }).collect() } } +fn gen_bool() -> bool { + rand::thread_rng().gen() +} + fn gen_string() -> String { const MIN_LEN: usize = 2; const MAX_LEN: usize = 5; @@ -90,22 +128,36 @@ fn test_client(test_fields: &TestFields) -> HttpBuffer { let request = ClientRequest::default(); - let mut test_files = test_fields.files.iter(); + let mut files = test_fields.files.iter(); + let mut nested_files = test_fields.nested.iter(); let mut multipart = Multipart::from_request(request).unwrap(); // Intersperse file fields amongst text fields for (name, text) in &test_fields.texts { - for (file_name, file) in &mut test_files { - multipart.write_stream(file_name, &mut &**file, Some(file_name), None).unwrap(); + if let Some((file_name, file)) = files.next() { + multipart.write_stream(file_name, &mut &*file.data, file.filename(), None) + .unwrap(); + } + + if let Some((file_name, files)) = nested_files.next() { + let (data, boundary) = gen_nested_multipart(files); + let mime = format!("multipart/mixed; boundary={}", boundary).parse().unwrap(); + multipart.write_stream(file_name, &mut &*data, None, Some(mime)).unwrap(); } multipart.write_text(name, text).unwrap(); } // Write remaining files - for (file_name, file) in test_files { - multipart.write_stream(file_name, &mut &**file, None, None).unwrap(); + for (file_name, file) in files { + multipart.write_stream(file_name, &mut &*file.data, None, None).unwrap(); + } + + for (file_name, files) in nested_files { + let (data, boundary) = gen_nested_multipart(files); + let mime = format!("multipart/mixed; boundary={}", boundary).parse().unwrap(); + multipart.write_stream(file_name, &mut &*data, None, Some(mime)).unwrap(); } multipart.send().unwrap() @@ -117,17 +169,32 @@ fn test_client_lazy(test_fields: &TestFields) -> HttpBuffer { let mut multipart = Multipart::new(); let mut test_files = test_fields.files.iter(); + let mut nested_files = test_fields.nested.iter(); for (name, text) in &test_fields.texts { for (file_name, file) in &mut test_files { - multipart.add_stream(&**file_name, Cursor::new(file), Some(&**file_name), None); + multipart.add_stream(&**file_name, Cursor::new(&file.data), file.filename(), None); + } + + if let Some((file_name, files)) = nested_files.next() { + let (data, boundary) = gen_nested_multipart(files); + let mime = format!("multipart/mixed; boundary={}", boundary).parse().unwrap(); + multipart.add_stream(&**file_name, Cursor::new(data), None as Option<&'static str>, + Some(mime)); } multipart.add_text(&**name, &**text); } for (file_name, file) in test_files { - multipart.add_stream(&**file_name, Cursor::new(file), None as Option<&str>, None); + multipart.add_stream(&**file_name, Cursor::new(&file.data), None as Option<&str>, None); + } + + for (file_name, files) in nested_files { + let (data, boundary) = gen_nested_multipart(files); + let mime = format!("multipart/mixed; boundary={}", boundary).parse().unwrap(); + multipart.add_stream(&**file_name, Cursor::new(data), None as Option<&'static str>, + Some(mime)); } let mut prepared = multipart.prepare_threshold(None).unwrap(); @@ -175,13 +242,14 @@ fn test_server(buf: HttpBuffer, mut fields: TestFields) { }, MultipartData::File(ref mut file) => { - let test_bytes = fields.files.remove(&field.name).unwrap(); + let test_file = fields.files.remove(&field.name).unwrap(); - let mut bytes = Vec::with_capacity(test_bytes.len()); + let mut bytes = Vec::with_capacity(test_file.data.len()); file.read_to_end(&mut bytes).unwrap(); - assert!(bytes == test_bytes, "Unexpected data for file {:?}: Expected {:?}, Got {:?}", - field.name, String::from_utf8_lossy(&test_bytes), String::from_utf8_lossy(&bytes) + assert!(bytes == test_file.data, "Unexpected data for file {:?}: Expected {:?}, Got {:?}", + field.name, String::from_utf8_lossy(&test_file.data), + String::from_utf8_lossy(&bytes) ); }, _ => unimplemented!(), @@ -192,5 +260,35 @@ fn test_server(buf: HttpBuffer, mut fields: TestFields) { assert!(fields.files.is_empty(), "File fields were not exhausted! File fields: {:?}", fields.files); } +fn gen_nested_multipart(files: &[FileEntry]) -> (Vec, String) { + let mut out = Vec::new(); + let boundary = gen_string(); + + write!(out, "Content-Type: multipart/mixed; boundary={boundary}\r\n\r\n \ + --{boundary}\r\n", boundary=boundary); + + let mut written = false; + + for file in files { + if written { + write!(out, "\r\n--{}\r\n", boundary); + } + + write!(out, "Content-Type: application/octet-stream"); + + if let Some(ref filename) = file.filename { + write!(out, "; filename={}", filename); + } + + write!(out, "\r\n\r\n"); + + out.write_all(&file.data); + written = true; + } + + write!(out, "\r\n--{}--\r\n", boundary); + + (out, boundary) +} From be340c294f2aeed738c0f7b0a32332983934b338 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sat, 4 Feb 2017 11:18:17 -0800 Subject: [PATCH 231/453] Fix Nested API --- multipart/src/server/field.rs | 125 ++++++++++++++++++++++------------ 1 file changed, 80 insertions(+), 45 deletions(-) diff --git a/multipart/src/server/field.rs b/multipart/src/server/field.rs index 7fba811c0..8c352388c 100644 --- a/multipart/src/server/field.rs +++ b/multipart/src/server/field.rs @@ -39,60 +39,67 @@ const EMPTY_STR_HEADER: StrHeader<'static> = StrHeader { val: "", }; +/// Not exposed #[derive(Copy, Clone, Debug)] -struct StrHeader<'a> { +pub struct StrHeader<'a> { name: &'a str, val: &'a str, } -/// The headers that (may) appear before a `multipart/form-data` field. -pub struct FieldHeaders { - /// The `Content-Disposition` header, required. - cont_disp: ContentDisp, - /// The `Content-Type` header, optional. - cont_type: Option, -} -impl FieldHeaders { - /// Parse the field headers from the passed `BufRead`, consuming the relevant bytes. - fn read_from(r: &mut R) -> io::Result> { - const HEADER_LEN: usize = 4; +fn with_headers(r: &mut R, f: F) -> io::Result +where R: BufRead, F: FnOnce(&[StrHeader]) -> Ret { + const HEADER_LEN: usize = 4; - // These are only written once so they don't need to be `mut` or initialized. - let consume; - let header_len; + // These are only written once so they don't need to be `mut` or initialized. + let consume; + let header_len; - let mut headers = [EMPTY_STR_HEADER; HEADER_LEN]; + let mut headers = [EMPTY_STR_HEADER; HEADER_LEN]; - { - let mut raw_headers = [EMPTY_HEADER; HEADER_LEN]; + { + let mut raw_headers = [EMPTY_HEADER; HEADER_LEN]; - loop { - let buf = try!(r.fill_buf()); + loop { + let buf = try!(r.fill_buf()); - match try_io!(httparse::parse_headers(buf, &mut raw_headers)) { - Status::Complete((consume_, raw_headers)) => { - consume = consume_; - header_len = raw_headers.len(); - break; - }, - Status::Partial => (), - } + match try_io!(httparse::parse_headers(buf, &mut raw_headers)) { + Status::Complete((consume_, raw_headers)) => { + consume = consume_; + header_len = raw_headers.len(); + break; + }, + Status::Partial => (), } + } - for (raw, header) in raw_headers.iter().take(header_len).zip(&mut headers) { - header.name = raw.name; - header.val = try!(io_str_utf8(raw.value)); - } + for (raw, header) in raw_headers.iter().take(header_len).zip(&mut headers) { + header.name = raw.name; + header.val = try!(io_str_utf8(raw.value)); } + } - let headers = &headers[..header_len]; + r.consume(consume); - debug!("Parsed field headers: {:?}", headers); + let headers = &headers[..header_len]; - r.consume(consume); + debug!("Parsed headers: {:?}", headers); - Ok(Self::parse(headers)) + Ok(f(headers)) +} + +/// The headers that (may) appear before a `multipart/form-data` field. +pub struct FieldHeaders { + /// The `Content-Disposition` header, required. + cont_disp: ContentDisp, + /// The `Content-Type` header, optional. + cont_type: Option, +} + +impl FieldHeaders { + /// Parse the field headers from the passed `BufRead`, consuming the relevant bytes. + fn read_from(r: &mut R) -> io::Result> { + with_headers(r, Self::parse) } fn parse(headers: &[StrHeader]) -> Option { @@ -548,6 +555,12 @@ fn create_full_path(path: &Path) -> io::Result { fs::File::create(&path) } +pub struct NestedEntry { + pub content_type: Mime, + pub filename: Option, + inner: M +} + #[derive(Debug)] pub struct NestedMultipart { outer_boundary: Vec, @@ -555,8 +568,30 @@ pub struct NestedMultipart { } impl NestedMultipart { - pub fn read_entry(&mut self) -> ReadEntryResult<&mut M> where for<'a> &'a mut M: ReadEntry { - self.inner.as_mut().expect("NestedMultipart::inner taken!()").read_entry() + pub fn read_entry(&mut self) -> ReadEntryResult<&mut M, NestedEntry<&mut M>> + where for<'a> &'a mut M: ReadEntry { + + let inner = self.inner_mut(); + + let headers = match inner.read_headers() { + Ok(Some(headers)) => headers, + Ok(None) => return End(inner), + Err(e) => return Error(inner, e), + }; + + let cont_type = match headers.cont_type { + Some(cont_type) => cont_type, + None => return ReadEntryResult::invalid_data(inner, + "Nested multipart requires Content-Type".to_string()) + }; + + Entry ( + NestedEntry { + filename: headers.cont_disp.filename, + content_type: cont_type, + inner: inner, + } + ) } fn inner_mut(&mut self) -> &mut M { @@ -666,30 +701,30 @@ impl ReadEntry for Multipart { impl<'a, M: ReadEntry> ReadEntry for &'a mut M { fn read_headers(&mut self) -> io::Result> { - self.read_headers() + (**self).read_headers() } fn read_to_string(&mut self) -> io::Result { - self.read_to_string() + (**self).read_to_string() } fn swap_boundary>>(&mut self, boundary: B) -> Vec { - self.swap_boundary(boundary) + (**self).swap_boundary(boundary) } } /// Result type returned by `Multipart::into_entry()` and `MultipartField::next_entry()`. -pub enum ReadEntryResult { +pub enum ReadEntryResult> { /// The next entry was found. - Entry(MultipartField), + Entry(Entry), /// No more entries could be read. End(M), /// An error occurred. Error(M, io::Error), } -impl ReadEntryResult { - pub fn into_result(self) -> io::Result>> { +impl ReadEntryResult { + pub fn into_result(self) -> io::Result> { match self { ReadEntryResult::Entry(entry) => Ok(Some(entry)), ReadEntryResult::End(_) => Ok(None), From f7deade81a022f16b57d52bbd16be32581a1e7b3 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Mon, 20 Feb 2017 04:34:38 -0800 Subject: [PATCH 232/453] `std::io::copy()` already continues on interrupt --- multipart/src/server/field.rs | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/multipart/src/server/field.rs b/multipart/src/server/field.rs index 8c352388c..c77c15967 100644 --- a/multipart/src/server/field.rs +++ b/multipart/src/server/field.rs @@ -377,7 +377,7 @@ impl MultipartFile where MultipartFile: Read { /// /// Retries when `io::Error::kind() == io::ErrorKind::Interrupted`. pub fn save_to(&mut self, mut out: W) -> io::Result { - retry_on_interrupt(|| io::copy(self, &mut out)) + io::copy(self, &mut out) } /// Save this file to the given output stream, **truncated** to `limit` @@ -387,7 +387,7 @@ impl MultipartFile where MultipartFile: Read { /// /// Retries when `io::Error::kind() == io::ErrorKind::Interrupted`. pub fn save_to_limited(&mut self, mut out: W, limit: u64) -> io::Result { - retry_on_interrupt(|| io::copy(&mut self.by_ref().take(limit), &mut out)) + io::copy(&mut self.by_ref().take(limit), &mut out) } /// Save this file to `path`. @@ -533,17 +533,6 @@ fn find_header<'a, 'b>(headers: &'a [StrHeader<'b>], name: &str) -> Option<&'a S headers.iter().find(|header| header.name == name) } -fn retry_on_interrupt(mut do_fn: F) -> io::Result where F: FnMut() -> io::Result { - loop { - match do_fn() { - Ok(val) => return Ok(val), - Err(err) => if err.kind() != io::ErrorKind::Interrupted { - return Err(err); - }, - } - } -} - fn create_full_path(path: &Path) -> io::Result { if let Some(parent) = path.parent() { try!(fs::create_dir_all(parent)); From 50fa01d59ee33a900199fde4e8373bb78e5eec89 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Mon, 20 Feb 2017 07:51:52 -0800 Subject: [PATCH 233/453] Use a builder type for file saving --- multipart/src/server/field.rs | 221 +++++++++++++++++++++++++++++----- multipart/src/server/iron.rs | 2 +- multipart/src/server/mod.rs | 6 +- 3 files changed, 192 insertions(+), 37 deletions(-) diff --git a/multipart/src/server/field.rs b/multipart/src/server/field.rs index c77c15967..7d68bc37a 100644 --- a/multipart/src/server/field.rs +++ b/multipart/src/server/field.rs @@ -12,13 +12,15 @@ use super::httparse::{self, EMPTY_HEADER, Status}; use super::Multipart; use self::ReadEntryResult::*; +use super::buf_redux::copy_buf; + use mime::{Attr, TopLevel, Mime, Value}; +use std::fs::{self, OpenOptions}; use std::io::{self, Read, BufRead, Write}; -use std::fs; use std::ops::Deref; use std::path::{Path, PathBuf}; -use std::{mem, str}; +use std::{env, mem, str}; const RANDOM_FILENAME_LEN: usize = 12; @@ -370,14 +372,20 @@ impl MultipartFile { } } -impl MultipartFile where MultipartFile: Read { +impl MultipartFile where MultipartFile: BufRead { + /// Get a builder type which can save the file with or without a size limit. + pub fn save(&mut self) -> SaveBuilder { + SaveBuilder::from_file(self) + } + /// Save this file to the given output stream. /// /// If successful, returns the number of bytes written. /// /// Retries when `io::Error::kind() == io::ErrorKind::Interrupted`. - pub fn save_to(&mut self, mut out: W) -> io::Result { - io::copy(self, &mut out) + #[deprecated = "use `.save().write_to()` instead"] + pub fn save_to(&mut self, out: W) -> io::Result { + self.save().write_to(out).map(|(size, _)| size) } /// Save this file to the given output stream, **truncated** to `limit` @@ -386,8 +394,9 @@ impl MultipartFile where MultipartFile: Read { /// If successful, returns the number of bytes written. /// /// Retries when `io::Error::kind() == io::ErrorKind::Interrupted`. - pub fn save_to_limited(&mut self, mut out: W, limit: u64) -> io::Result { - io::copy(&mut self.by_ref().take(limit), &mut out) + #[deprecated = "use `.save().limit(limit).write_to(out)` instead"] + pub fn save_to_limited(&mut self, out: W, limit: u64) -> io::Result { + self.save().limit(limit).write_to(out).map(|(size, _)| size) } /// Save this file to `path`. @@ -395,16 +404,9 @@ impl MultipartFile where MultipartFile: Read { /// Returns the saved file info on success, or any errors otherwise. /// /// Retries when `io::Error::kind() == io::ErrorKind::Interrupted`. + #[deprecated = "use `.save().with_path(path)` instead"] pub fn save_as>(&mut self, path: P) -> io::Result { - let path = path.into(); - let file = try!(create_full_path(&path)); - let size = try!(self.save_to(file)); - - Ok(SavedFile { - path: path, - filename: self.filename.clone(), - size: size, - }) + self.save().with_path(path) } /// Save this file in the directory pointed at by `dir`, @@ -415,9 +417,9 @@ impl MultipartFile where MultipartFile: Read { /// Returns the saved file's info on success, or any errors otherwise. /// /// Retries when `io::Error::kind() == io::ErrorKind::Interrupted`. + #[deprecated = "use `.save().with_dir(dir)` instead"] pub fn save_in>(&mut self, dir: P) -> io::Result { - let path = dir.as_ref().join(::random_alphanumeric(RANDOM_FILENAME_LEN)); - self.save_as(path) + self.save().with_dir(dir.as_ref()) } /// Save this file to `path`, **truncated** to `limit` (no more than `limit` bytes will be written out). @@ -427,16 +429,9 @@ impl MultipartFile where MultipartFile: Read { /// Returns the saved file's info on success, or any errors otherwise. /// /// Retries when `io::Error::kind() == io::ErrorKind::Interrupted`. + #[deprecated = "use `.save().limit(limit).with_path(path)` instead"] pub fn save_as_limited>(&mut self, path: P, limit: u64) -> io::Result { - let path = path.into(); - let file = try!(create_full_path(&path)); - let size = try!(self.save_to_limited(file, limit)); - - Ok(SavedFile { - path: path, - filename: self.filename.clone(), - size: size, - }) + self.save().limit(limit).with_path(path) } /// Save this file in the directory pointed at by `dir`, @@ -449,9 +444,9 @@ impl MultipartFile where MultipartFile: Read { /// Returns the saved file's info on success, or any errors otherwise. /// /// Retries when `io::Error::kind() == io::ErrorKind::Interrupted`. + #[deprecated = "use `.save().limit(limit).with_dir(dir)` instead"] pub fn save_in_limited>(&mut self, dir: P, limit: u64) -> io::Result { - let path = dir.as_ref().join(::random_alphanumeric(RANDOM_FILENAME_LEN)); - self.save_as_limited(path, limit) + self.save().limit(limit).with_dir(dir) } } @@ -487,6 +482,149 @@ impl<'a, R: Read + 'a> BufRead for MultipartFile<&'a mut Multipart> { } } +/// A builder for saving a `MultipartFile` to the local filesystem. +pub struct SaveBuilder<'m, M: 'm> { + file: &'m mut MultipartFile, + open_opts: OpenOptions, + limit: Option, +} + +impl<'m, M: 'm> SaveBuilder<'m, M> where MultipartFile: BufRead { + fn from_file(file: &'m mut MultipartFile) -> Self { + let mut open_opts = OpenOptions::new(); + open_opts.write(true).create_new(true); + + SaveBuilder { + file: file, + open_opts: open_opts, + limit: None, + } + } + + /// Set the maximum number of bytes to write out. + /// + /// Can be `u64` or `Option`. If `None`, clears the limit. + pub fn limit>>(&mut self, limit: L) -> &mut Self { + self.limit = limit.into(); + self + } + + /// Modify the `OpenOptions` used to open the file for writing. + /// + /// The `write` flag will be reset to `true` after the closure returns. (It'd be pretty + /// pointless otherwise, right?) + pub fn mod_open_opts(&mut self, opts_fn: F) -> &mut Self { + opts_fn(&mut self.open_opts); + self.open_opts.write(true); + self + } + + /// Save to a file with a random alphanumeric name in the given directory. + /// + /// See `with_path()` for more details. + /// + /// ### Warning: Do **not* trust user input! + /// It is a serious security risk to create files or directories with paths based on user input. + /// A malicious user could craft a path which can be used to overwrite important files, such as + /// web templates, static assets, Javascript files, database files, configuration files, etc., + /// if they are writable by the server process. + /// + /// This can be mitigated somewhat by setting filesystem permissions as + /// conservatively as possible and running the server under its own user with restricted + /// permissions, but you should still not use user input directly as filesystem paths. + /// If it is truly necessary, you should sanitize filenames such that they cannot be + /// misinterpreted by the OS. + pub fn with_dir>(&mut self, dir: P) -> io::Result { + let path = dir.as_ref().join(rand_filename()); + self.with_path(path) + } + + /// Save to a file with the given name in the OS temporary directory. + /// + /// See `with_path()` for more details. + /// + /// ### Warning: Do **not* trust user input! + /// It is a serious security risk to create files or directories with paths based on user input. + /// A malicious user could craft a path which can be used to overwrite important files, such as + /// web templates, static assets, Javascript files, database files, configuration files, etc., + /// if they are writable by the server process. + /// + /// This can be mitigated somewhat by setting filesystem permissions as + /// conservatively as possible and running the server under its own user with restricted + /// permissions, but you should still not use user input directly as filesystem paths. + /// If it is truly necessary, you should sanitize filenames such that they cannot be + /// misinterpreted by the OS. + pub fn with_filename>(&mut self, filename: P) -> io::Result { + self.with_path(env::temp_dir().join(filename)) + } + + /// Save to a file with the given path. + /// + /// Truncates the file to the given limit, if set, and the contained `OpenOptions`. + /// + /// ### Warning: Do **not* trust user input! + /// It is a serious security risk to create files or directories with paths based on user input. + /// A malicious user could craft a path which can be used to overwrite important files, such as + /// web templates, static assets, Javascript files, database files, configuration files, etc., + /// if they are writable by the server process. + /// + /// This can be mitigated somewhat by setting filesystem permissions as + /// conservatively as possible and running the server under its own user with restricted + /// permissions, but you should still not use user input directly as filesystem paths. + /// If it is truly necessary, you should sanitize filenames such that they cannot be + /// misinterpreted by the OS. + /// + /// ### `OpenOptions` + /// By default, the open options are set with `.write(true).create_new(true)`, + /// so if the file already exists then an error will be thrown. This is to avoid accidentally + /// overwriting files from other requests. + /// + /// If you want to modify the options used to open the save file, you can use + /// `mod_open_options()`. + pub fn with_path>(&mut self, path: P) -> io::Result { + let path = path.into(); + + try!(create_full_path(&path)); + + let mut file = try!(self.open_opts.open(&path)); + let (written, truncated) = try!(self.write_to(file)); + + Ok(SavedFile { + path: path, + filename: self.file.filename.clone(), + size: written, + truncated: truncated, + _priv: (), + }) + } + + /// Save to a file with a random alphanumeric name in the OS temporary directory. + /// + /// Does not use user input to create the path. + /// + /// See `with_path()` for more details. + pub fn temp(&mut self) -> io::Result { + let path = env::temp_dir().join(rand_filename()); + self.with_path(path) + } + + /// Write out the file field to `dest`, truncating if a limit was set. + /// + /// Returns the number of bytes copied, and whether or not the limit was reached + /// (tested by `MultipartFile::fill_buf().is_empty()` so no bytes are consumed). + /// + /// Retries on interrupts. + pub fn write_to(&mut self, mut dest: W) -> io::Result<(u64, bool)> { + if let Some(limit) = self.limit { + let copied = try!(copy_buf(&mut self.file.by_ref().take(limit), &mut dest)); + // If there's more data to be read, the field was truncated + Ok((copied, !try!(self.file.fill_buf()).is_empty())) + } else { + copy_buf(&mut self.file, &mut dest).map(|copied| (copied, false)) + } + } +} + /// A file saved to the local filesystem from a multipart request. #[derive(Debug)] pub struct SavedFile { @@ -495,14 +633,31 @@ pub struct SavedFile { /// The original filename of this file, if one was provided in the request. /// - /// ##Warning + /// ##Warning: Provided by Client! Do **not** trust user input! /// You should treat this value as untrustworthy because it is an arbitrary string provided by /// the client. You should *not* blindly append it to a directory path and save the file there, /// as such behavior could easily be exploited by a malicious client. + /// + /// It is a serious security risk to create files or directories with paths based on user input. + /// A malicious user could craft a path which can be used to overwrite important files, such as + /// web templates, static assets, Javascript files, database files, configuration files, etc., + /// if they are writable by the server process. + /// + /// This can be mitigated somewhat by setting filesystem permissions as + /// conservatively as possible and running the server under its own user with restricted + /// permissions, but you should still not use user input directly as filesystem paths. + /// If it is truly necessary, you should sanitize filenames such that they cannot be + /// misinterpreted by the OS. pub filename: Option, - /// The number of bytes written to the disk; may be truncated. + /// The number of bytes written to the disk. May be truncated, check the `truncated` flag + /// before making any assumptions based on this number. pub size: u64, + + /// If the file save limit was hit and the saved file ended up truncated. + pub truncated: bool, + // Private field to prevent exhaustive matching for backwards compatibility + _priv: (), } fn read_content_type(cont_type: &str) -> Mime { @@ -544,6 +699,10 @@ fn create_full_path(path: &Path) -> io::Result { fs::File::create(&path) } +fn rand_filename() -> String { + ::random_alphanumeric(RANDOM_FILENAME_LEN) +} + pub struct NestedEntry { pub content_type: Mime, pub filename: Option, diff --git a/multipart/src/server/iron.rs b/multipart/src/server/iron.rs index 7643cff09..32f016e06 100644 --- a/multipart/src/server/iron.rs +++ b/multipart/src/server/iron.rs @@ -150,7 +150,7 @@ impl Intercept { } let file = try_iron!( - file.save_in_limited(&entries.dir, self.file_size_limit); + file.save().limit(self.file_size_limit).with_dir(&entries.dir); "Error reading field: \"{}\" (filename: \"{}\")", field.name, file.filename().unwrap_or("(none)") diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index 0e8559811..70afc7474 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -218,11 +218,7 @@ impl Multipart { while let Some(field) = try!(self.read_entry()) { match field.data { MultipartData::File(mut file) => { - let file = if let Some(limit) = limit { - try!(file.save_in_limited(&entries.dir, limit)) - } else { - try!(file.save_in(&entries.dir)) - }; + let file = try!(file.save().limit(limit).with_dir(&entries.dir)); entries.files.insert(field.name, file); }, From 915623176d12ee1499417d1b28a8a9f8c679d69b Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Tue, 21 Feb 2017 13:10:30 -0800 Subject: [PATCH 234/453] Fix bugs in `next_entry()`; export `ReadEntry` as an empty trait with unaccessible super; add entry API to local_test --- multipart/src/local_test.rs | 181 +++++++++++++++++++++---------- multipart/src/server/boundary.rs | 3 +- multipart/src/server/field.rs | 117 ++++++++++++-------- multipart/src/server/mod.rs | 47 ++++---- 4 files changed, 220 insertions(+), 128 deletions(-) diff --git a/multipart/src/local_test.rs b/multipart/src/local_test.rs index c5472bf82..d5aab9353 100644 --- a/multipart/src/local_test.rs +++ b/multipart/src/local_test.rs @@ -6,9 +6,13 @@ // copied, modified, or distributed except according to those terms. use mock::{ClientRequest, HttpBuffer}; +use server::{MultipartField, MultipartData, ReadEntry}; + use rand::{self, Rng}; use std::collections::HashMap; +use std::convert::AsRef; +use std::fmt; use std::io::prelude::*; use std::io::Cursor; use std::iter; @@ -20,10 +24,51 @@ struct TestFields { nested: HashMap>, } +impl TestFields { + fn check_field(&mut self, field: &mut MultipartField) { + match field.data { + MultipartData::Text(ref text) => { + let test_text = self.texts.remove(&field.name); + + assert!( + test_text.is_some(), + "Got text field that wasn't in original dataset: {:?} : {:?} ", + field.name, text.text + ); + + let test_text = test_text.unwrap(); + + assert!( + text.text == test_text, + "Unexpected data for field {:?}: Expected {:?}, got {:?}", + field.name, test_text, text.text + ); + }, + MultipartData::File(ref mut file) => { + let test_file = self.files.remove(&field.name).unwrap(); + + let mut bytes = Vec::with_capacity(test_file.data.0.len()); + file.read_to_end(&mut bytes).unwrap(); + + assert!(bytes == test_file.data.0, "Unexpected data for file {:?}: Expected {:?}, Got {:?}", + field.name, String::from_utf8_lossy(&test_file.data.0), + String::from_utf8_lossy(&bytes) + ); + }, + _ => (), + } + } + + fn assert_is_empty(&self) { + assert!(self.texts.is_empty(), "Text fields were not exhausted! Text fields: {:?}", self.texts); + assert!(self.files.is_empty(), "File fields were not exhausted! File fields: {:?}", self.files); + } +} + #[derive(Debug)] struct FileEntry { filename: Option, - data: Vec, + data: PrintHex, } impl FileEntry { @@ -35,7 +80,7 @@ impl FileEntry { FileEntry { filename: filename, - data: gen_bytes() + data: PrintHex(gen_bytes()) } } @@ -44,33 +89,70 @@ impl FileEntry { } } -#[test] -fn local_test() { - do_test(test_client, "Regular"); -} +struct PrintHex(Vec); -#[test] -fn local_test_lazy() { - do_test(test_client_lazy, "Lazy"); +impl fmt::Debug for PrintHex { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + try!(write!(f, "[")); + + let mut written = false; + + for byte in &self.0 { + try!(write!(f, "{:X}", byte)); + + if written { + try!(write!(f, ", ")); + } + + written = true; + } + + write!(f, "]") + } } -fn do_test(client: fn(&TestFields) -> HttpBuffer, name: &str) { - let _ = ::env_logger::init(); +macro_rules! do_test ( + ($client_test:ident, $server_test:ident) => ( + let _ = ::env_logger::init(); - info!("Testing {} client", name); + info!("Client Test: {:?} Server Test: {:?}", stringify!($client_test), + stringify!($server_test)); - let test_fields = gen_test_fields(); + let mut test_fields = gen_test_fields(); - trace!("Fields for test: {:?}", test_fields); + trace!("Fields for test: {:?}", test_fields); - let buf = client(&test_fields); + let buf = $client_test(&test_fields); - trace!( - "\n==Test Buffer Begin==\n{}\n==Test Buffer End==", - String::from_utf8_lossy(&buf.buf) + trace!( + "\n==Test Buffer Begin==\n{}\n==Test Buffer End==", + String::from_utf8_lossy(&buf.buf) + ); + + $server_test(buf, &mut test_fields); + + test_fields.assert_is_empty(); ); +); + +#[test] +fn reg_client_reg_server() { + do_test!(test_client, test_server); +} - test_server(buf, test_fields); +#[test] +fn reg_client_entry_server() { + do_test!(test_client, test_server_entry_api); +} + +#[test] +fn lazy_client_reg_server() { + do_test!(test_client_lazy, test_server); +} + +#[test] +fn lazy_client_entry_server() { + do_test!(test_client_lazy, test_server_entry_api); } fn gen_test_fields() -> TestFields { @@ -136,7 +218,7 @@ fn test_client(test_fields: &TestFields) -> HttpBuffer { // Intersperse file fields amongst text fields for (name, text) in &test_fields.texts { if let Some((file_name, file)) = files.next() { - multipart.write_stream(file_name, &mut &*file.data, file.filename(), None) + multipart.write_stream(file_name, &mut &*file.data.0, file.filename(), None) .unwrap(); } @@ -151,7 +233,7 @@ fn test_client(test_fields: &TestFields) -> HttpBuffer { // Write remaining files for (file_name, file) in files { - multipart.write_stream(file_name, &mut &*file.data, None, None).unwrap(); + multipart.write_stream(file_name, &mut &*file.data.0, None, None).unwrap(); } for (file_name, files) in nested_files { @@ -173,7 +255,7 @@ fn test_client_lazy(test_fields: &TestFields) -> HttpBuffer { for (name, text) in &test_fields.texts { for (file_name, file) in &mut test_files { - multipart.add_stream(&**file_name, Cursor::new(&file.data), file.filename(), None); + multipart.add_stream(&**file_name, Cursor::new(&file.data.0), file.filename(), None); } if let Some((file_name, files)) = nested_files.next() { @@ -187,7 +269,7 @@ fn test_client_lazy(test_fields: &TestFields) -> HttpBuffer { } for (file_name, file) in test_files { - multipart.add_stream(&**file_name, Cursor::new(&file.data), None as Option<&str>, None); + multipart.add_stream(&**file_name, Cursor::new(&file.data.0), None as Option<&str>, None); } for (file_name, files) in nested_files { @@ -209,8 +291,8 @@ fn test_client_lazy(test_fields: &TestFields) -> HttpBuffer { HttpBuffer::with_buf(buf, boundary, content_len) } -fn test_server(buf: HttpBuffer, mut fields: TestFields) { - use server::{Multipart, MultipartData}; +fn test_server(buf: HttpBuffer, fields: &mut TestFields) { + use server::Multipart; let server_buf = buf.for_server(); @@ -221,43 +303,30 @@ fn test_server(buf: HttpBuffer, mut fields: TestFields) { let mut multipart = Multipart::from_request(server_buf) .unwrap_or_else(|_| panic!("Buffer should be multipart!")); - while let Ok(Some(mut field)) = multipart.read_entry() { - match field.data { - MultipartData::Text(text) => { - let test_text = fields.texts.remove(&field.name); + while let Some(mut field) = multipart.read_entry().unwrap() { + fields.check_field(&mut field); + } +} - assert!( - test_text.is_some(), - "Got text field that wasn't in original dataset: {:?} : {:?} ", - field.name, text.text - ); +fn test_server_entry_api(buf: HttpBuffer, fields: &mut TestFields) { + use server::Multipart; - let test_text = test_text.unwrap(); + let server_buf = buf.for_server(); - assert!( - text.text == test_text, - "Unexpected data for field {:?}: Expected {:?}, got {:?}", - field.name, test_text, text.text - ); + if let Some(content_len) = server_buf.content_len { + assert!(content_len == server_buf.data.len() as u64, "Supplied content_len different from actual"); + } - }, - MultipartData::File(ref mut file) => { - let test_file = fields.files.remove(&field.name).unwrap(); + let multipart = Multipart::from_request(server_buf) + .unwrap_or_else(|_| panic!("Buffer should be multipart!")); - let mut bytes = Vec::with_capacity(test_file.data.len()); - file.read_to_end(&mut bytes).unwrap(); + let mut entry = multipart.into_entry().expect_alt("Expected entry, got none", "Error reading entry"); + fields.check_field(&mut entry); - assert!(bytes == test_file.data, "Unexpected data for file {:?}: Expected {:?}, Got {:?}", - field.name, String::from_utf8_lossy(&test_file.data), - String::from_utf8_lossy(&bytes) - ); - }, - _ => unimplemented!(), - } + while let Some(entry_) = entry.next_entry().unwrap_opt() { + entry = entry_; + fields.check_field(&mut entry); } - - assert!(fields.texts.is_empty(), "Text fields were not exhausted! Text fields: {:?}", fields.texts); - assert!(fields.files.is_empty(), "File fields were not exhausted! File fields: {:?}", fields.files); } fn gen_nested_multipart(files: &[FileEntry]) -> (Vec, String) { @@ -282,7 +351,7 @@ fn gen_nested_multipart(files: &[FileEntry]) -> (Vec, String) { write!(out, "\r\n\r\n"); - out.write_all(&file.data); + out.write_all(&file.data.0); written = true; } diff --git a/multipart/src/server/boundary.rs b/multipart/src/server/boundary.rs index 95c66b044..4a8710ef8 100644 --- a/multipart/src/server/boundary.rs +++ b/multipart/src/server/boundary.rs @@ -132,8 +132,7 @@ impl BoundaryReader where R: Read { if log_enabled!(LogLevel::Trace) { trace!("Consumed up to self.search_idx, remaining buf: {:?}", - String::from_utf8_lossy(self.source.get_buf()) - ); + String::from_utf8_lossy(self.source.get_buf())); } let consume_amt = { diff --git a/multipart/src/server/field.rs b/multipart/src/server/field.rs index 7d68bc37a..2d575a1ca 100644 --- a/multipart/src/server/field.rs +++ b/multipart/src/server/field.rs @@ -10,6 +10,7 @@ use super::httparse::{self, EMPTY_HEADER, Status}; use super::Multipart; + use self::ReadEntryResult::*; use super::buf_redux::copy_buf; @@ -372,7 +373,7 @@ impl MultipartFile { } } -impl MultipartFile where MultipartFile: BufRead { +impl MultipartFile where M: ReadEntry { /// Get a builder type which can save the file with or without a size limit. pub fn save(&mut self) -> SaveBuilder { SaveBuilder::from_file(self) @@ -450,35 +451,19 @@ impl MultipartFile where MultipartFile: BufRead { } } -impl Read for MultipartFile> { - fn read(&mut self, buf: &mut [u8]) -> io::Result{ - self.inner_mut().reader.read(buf) - } -} - -impl BufRead for MultipartFile> { - fn fill_buf(&mut self) -> io::Result<&[u8]> { - self.inner_mut().reader.fill_buf() - } - - fn consume(&mut self, amt: usize) { - self.inner_mut().reader.consume(amt) - } -} - -impl<'a, R: Read + 'a> Read for MultipartFile<&'a mut Multipart> { +impl Read for MultipartFile { fn read(&mut self, buf: &mut [u8]) -> io::Result{ - self.inner_mut().reader.read(buf) + self.inner_mut().source().read(buf) } } -impl<'a, R: Read + 'a> BufRead for MultipartFile<&'a mut Multipart> { +impl BufRead for MultipartFile { fn fill_buf(&mut self) -> io::Result<&[u8]> { - self.inner_mut().reader.fill_buf() + self.inner_mut().source().fill_buf() } fn consume(&mut self, amt: usize) { - self.inner_mut().reader.consume(amt) + self.inner_mut().source().consume(amt) } } @@ -770,15 +755,41 @@ impl Drop for NestedMultipart { } } +/// Common trait for `Multipart` and `&mut Multipart` +pub trait ReadEntry: PrivReadEntry {} + +impl ReadEntry for T where T: PrivReadEntry {} + /// Public trait but not re-exported. -pub trait ReadEntry: Sized { - fn read_headers(&mut self) -> io::Result>; +pub trait PrivReadEntry: Sized { + type Source: BufRead; - fn read_to_string(&mut self) -> io::Result; + fn source(&mut self) -> &mut Self::Source; + + /// Consume the next boundary. + /// Returns `true` if the last boundary was read, `false` otherwise. + fn consume_boundary(&mut self) -> io::Result; fn swap_boundary>>(&mut self, boundary: B) -> Vec; + fn read_headers(&mut self) -> io::Result> { + FieldHeaders::read_from(&mut self.source()) + } + + fn read_to_string(&mut self) -> io::Result { + let mut buf = String::new(); + + match self.source().read_to_string(&mut buf) { + Ok(_) => Ok(buf), + Err(err) => Err(err), + } + } + fn read_entry(mut self) -> ReadEntryResult { + if try_read_entry!(self; self.consume_boundary()) { + return End(self); + } + let field_headers = match try_read_entry!(self; self.read_headers()) { Some(headers) => headers, None => return End(self), @@ -833,32 +844,19 @@ pub trait ReadEntry: Sized { } } -impl ReadEntry for Multipart { - fn read_headers(&mut self) -> io::Result> { - FieldHeaders::read_from(&mut self.reader) - } +impl<'a, M: ReadEntry> PrivReadEntry for &'a mut M { + type Source = M::Source; - fn read_to_string(&mut self) -> io::Result { - self.read_to_string() - } - - fn swap_boundary>>(&mut self, boundary: B) -> Vec { - self.reader.swap_boundary(boundary) - } -} - -impl<'a, M: ReadEntry> ReadEntry for &'a mut M { - fn read_headers(&mut self) -> io::Result> { - (**self).read_headers() - } - - fn read_to_string(&mut self) -> io::Result { - (**self).read_to_string() + fn source(&mut self) -> &mut M::Source { + (**self).source() } fn swap_boundary>>(&mut self, boundary: B) -> Vec { (**self).swap_boundary(boundary) } + fn consume_boundary(&mut self) -> io::Result { + (**self).consume_boundary() + } } /// Result type returned by `Multipart::into_entry()` and `MultipartField::next_entry()`. @@ -880,6 +878,35 @@ impl ReadEntryResult { } } + pub fn unwrap(self) -> Entry { + self.expect_alt("`ReadEntryResult::unwrap()` called on `End` value", + "`ReadEntryResult::unwrap()` called on `Error` value: {:?}") + } + + pub fn expect(self, msg: &str) -> Entry { + self.expect_alt(msg, msg) + } + + pub fn expect_alt(self, end_msg: &str, err_msg: &str) -> Entry { + match self { + Entry(entry) => entry, + End(_) => panic!("{}", end_msg), + Error(_, err) => panic!("{}: {:?}", err_msg, err), + } + } + + pub fn unwrap_opt(self) -> Option { + self.expect_opt("`ReadEntryResult::unwrap_opt()` called on `Error` value") + } + + pub fn expect_opt(self, msg: &str) -> Option { + match self { + Entry(entry) => Some(entry), + End(_) => None, + Error(_, err) => panic!("{}: {:?}", msg, err), + } + } + fn invalid_data(multipart: M, msg: String) -> Self { ReadEntryResult::Error ( multipart, diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index 70afc7474..b9e483b57 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -26,9 +26,11 @@ use std::path::{Path, PathBuf}; use std::{io, mem}; use self::boundary::BoundaryReader; -use self::field::ReadEntry; -pub use self::field::{MultipartField, MultipartFile, MultipartData, ReadEntryResult, SavedFile}; +use self::field::PrivReadEntry; + +pub use self::field::{MultipartField, MultipartFile, MultipartData, ReadEntry, ReadEntryResult, + SaveBuilder, SavedFile}; macro_rules! try_opt ( ($expr:expr) => ( @@ -109,20 +111,12 @@ impl Multipart { /// If the previously returned entry had contents of type `MultipartField::File`, /// calling this again will discard any unread contents of that entry. pub fn read_entry(&mut self) -> io::Result>> { - if try!(self.consume_boundary()) { - return Ok(None); - } - - ReadEntry::read_entry(self).into_result() + PrivReadEntry::read_entry(self).into_result() } /// Read the next entry from this multipart request, returning a struct with the field's name and /// data. See `MultipartField` for more info. - pub fn into_entry(mut self) -> ReadEntryResult { - if try_read_entry!(self; self.consume_boundary()) { - return ReadEntryResult::End(self); - } - + pub fn into_entry(self) -> ReadEntryResult { self.read_entry() } @@ -230,15 +224,24 @@ impl Multipart { } Ok(()) - } + } +} + +impl Borrow for Multipart { + fn borrow(&self) -> &R { + self.reader.borrow() + } +} - fn read_to_string(&mut self) -> io::Result { - let mut buf = String::new(); +impl PrivReadEntry for Multipart { + type Source = BoundaryReader; - match self.reader.read_to_string(&mut buf) { - Ok(_) => Ok(buf), - Err(err) => Err(err), - } + fn source(&mut self) -> &mut BoundaryReader { + &mut self.reader + } + + fn swap_boundary>>(&mut self, boundary: B) -> Vec { + self.reader.swap_boundary(boundary) } /// Consume the next boundary. @@ -249,12 +252,6 @@ impl Multipart { } } -impl Borrow for Multipart { - fn borrow(&self) -> &B { - self.reader.borrow() - } -} - /// The result of [`Multipart::save_all()`](struct.Multipart.html#method.save_all). #[derive(Debug)] pub enum SaveResult { From 4cb78bba95b6852ebb4022d10f76f1c731e9f57d Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Tue, 21 Feb 2017 13:11:08 -0800 Subject: [PATCH 235/453] Bump alpha version --- multipart/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 98f0b87d4..f9218e758 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "multipart" -version = "0.10.0-alpha.1" +version = "0.10.0-alpha.2" authors = ["Austin Bonander "] From da0c1ac7bc0e1bd4cf4a64c74e8f151808315a21 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Tue, 21 Feb 2017 13:29:43 -0800 Subject: [PATCH 236/453] Deny missing documentation; add missing documentation --- multipart/src/lib.rs | 2 +- multipart/src/server/field.rs | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/multipart/src/lib.rs b/multipart/src/lib.rs index 30ecdf7d6..6c1139651 100644 --- a/multipart/src/lib.rs +++ b/multipart/src/lib.rs @@ -28,7 +28,7 @@ //! * `nickel_`: Integration with the [Nickel](http://nickel.rs) web application framework. //! See the [`server::nickel`](server/nickel/index.html) module for more information. Enables the `hyper` //! feature. -#![warn(missing_docs)] +#![deny(missing_docs)] #![cfg_attr(feature = "nightly", feature(insert_str))] #[macro_use] diff --git a/multipart/src/server/field.rs b/multipart/src/server/field.rs index 2d575a1ca..aac8fb98c 100644 --- a/multipart/src/server/field.rs +++ b/multipart/src/server/field.rs @@ -870,6 +870,11 @@ pub enum ReadEntryResult> { } impl ReadEntryResult { + /// Convert `self` into `Result>` as follows: + /// + /// * `Entry(entry) -> Ok(Some(entry))` + /// * `End(_) -> Ok(None)` + /// * `Error(_, err) -> Err(err)` pub fn into_result(self) -> io::Result> { match self { ReadEntryResult::Entry(entry) => Ok(Some(entry)), @@ -878,15 +883,21 @@ impl ReadEntryResult { } } + /// Attempt to unwrap `Entry`, panicking if this is `End` or `Error`. pub fn unwrap(self) -> Entry { self.expect_alt("`ReadEntryResult::unwrap()` called on `End` value", "`ReadEntryResult::unwrap()` called on `Error` value: {:?}") } + /// Attempt to unwrap `Entry`, panicking if this is `End` or `Error` + /// with the given message. Adds the error's message in the `Error` case. pub fn expect(self, msg: &str) -> Entry { self.expect_alt(msg, msg) } + /// Attempt to unwrap `Entry`, panicking if this is `End` or `Error`. + /// If this is `End`, panics with `end_msg`; if `Error`, panics with `err_msg` + /// as well as the error's message. pub fn expect_alt(self, end_msg: &str, err_msg: &str) -> Entry { match self { Entry(entry) => entry, @@ -895,10 +906,13 @@ impl ReadEntryResult { } } + /// Attempt to unwrap as `Option`, panicking in the `Error` case. pub fn unwrap_opt(self) -> Option { self.expect_opt("`ReadEntryResult::unwrap_opt()` called on `Error` value") } + /// Attempt to unwrap as `Option`, panicking in the `Error` case + /// with the given message as well as the error's message. pub fn expect_opt(self, msg: &str) -> Option { match self { Entry(entry) => Some(entry), From ade4b3c1fb44354ab69ea6f782637541ff41fe64 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Tue, 21 Feb 2017 18:41:13 -0800 Subject: [PATCH 237/453] Remove prototype impl of nested fields and vestigial support; add test for multiple files per field name --- multipart/src/lib.rs | 3 +- multipart/src/local_test.rs | 213 ++++++++++++++----------------- multipart/src/server/boundary.rs | 15 --- multipart/src/server/field.rs | 177 ++++++++++--------------- multipart/src/server/iron.rs | 3 +- multipart/src/server/mod.rs | 5 - 6 files changed, 165 insertions(+), 251 deletions(-) diff --git a/multipart/src/lib.rs b/multipart/src/lib.rs index 6c1139651..b039d7552 100644 --- a/multipart/src/lib.rs +++ b/multipart/src/lib.rs @@ -29,7 +29,6 @@ //! See the [`server::nickel`](server/nickel/index.html) module for more information. Enables the `hyper` //! feature. #![deny(missing_docs)] -#![cfg_attr(feature = "nightly", feature(insert_str))] #[macro_use] extern crate log; @@ -37,7 +36,9 @@ extern crate log; #[cfg(test)] extern crate env_logger; +#[cfg_attr(test, macro_use)] extern crate mime; + extern crate mime_guess; extern crate rand; extern crate tempdir; diff --git a/multipart/src/local_test.rs b/multipart/src/local_test.rs index d5aab9353..dd43e022b 100644 --- a/multipart/src/local_test.rs +++ b/multipart/src/local_test.rs @@ -8,36 +8,59 @@ use mock::{ClientRequest, HttpBuffer}; use server::{MultipartField, MultipartData, ReadEntry}; +use mime::Mime; + use rand::{self, Rng}; -use std::collections::HashMap; -use std::convert::AsRef; +use std::collections::{HashMap, HashSet}; use std::fmt; use std::io::prelude::*; use std::io::Cursor; -use std::iter; +use std::iter::{self, FromIterator}; + +const MIN_FIELDS: usize = 1; +const MAX_FIELDS: usize = 3; + +const MIN_LEN: usize = 2; +const MAX_LEN: usize = 5; +const MAX_DASHES: usize = 2; + +fn collect_rand, T, F: FnMut() -> T>(mut gen: F) -> C { + (0 .. rand::thread_rng().gen_range(MIN_FIELDS, MAX_FIELDS)) + .map(|_| gen()).collect() +} + +macro_rules! expect_fmt ( + ($val:expr, $($args:tt)*) => ( + match $val { + Some(val) => val, + None => panic!($($args)*), + } + ); +); #[derive(Debug)] struct TestFields { texts: HashMap, - files: HashMap, - nested: HashMap>, + files: HashMap>, } impl TestFields { + fn gen() -> Self { + TestFields { + texts: collect_rand(|| (gen_string(), gen_string())), + files: collect_rand(|| (gen_string(), FileEntry::gen_many())), + } + } + fn check_field(&mut self, field: &mut MultipartField) { match field.data { MultipartData::Text(ref text) => { - let test_text = self.texts.remove(&field.name); - - assert!( - test_text.is_some(), + let test_text = expect_fmt!(self.texts.remove(&field.name), "Got text field that wasn't in original dataset: {:?} : {:?} ", field.name, text.text ); - let test_text = test_text.unwrap(); - assert!( text.text == test_text, "Unexpected data for field {:?}: Expected {:?}, got {:?}", @@ -45,17 +68,30 @@ impl TestFields { ); }, MultipartData::File(ref mut file) => { - let test_file = self.files.remove(&field.name).unwrap(); - - let mut bytes = Vec::with_capacity(test_file.data.0.len()); + let mut bytes = Vec::with_capacity(MAX_LEN); file.read_to_end(&mut bytes).unwrap(); - assert!(bytes == test_file.data.0, "Unexpected data for file {:?}: Expected {:?}, Got {:?}", - field.name, String::from_utf8_lossy(&test_file.data.0), - String::from_utf8_lossy(&bytes) - ); + let curr_file = FileEntry { + content_type: file.content_type.clone(), + filename: file.filename.take(), + data: PrintHex(bytes), + }; + + let files_empty = { + let mut files = expect_fmt!(self.files.get_mut(&field.name), + "Got file field that wasn't in original dataset: {:?} : {:?}", + field.name, curr_file); + + assert!(files.remove(&curr_file), "Unexpected data for file field {:?}: {:?}", + field.name, curr_file); + + files.is_empty() + }; + + if files_empty { + let _ = self.files.remove(&field.name); + } }, - _ => (), } } @@ -65,13 +101,18 @@ impl TestFields { } } -#[derive(Debug)] +#[derive(Debug, Hash, PartialEq, Eq)] struct FileEntry { + content_type: Mime, filename: Option, data: PrintHex, } impl FileEntry { + fn gen_many() -> HashSet { + collect_rand(Self::gen) + } + fn gen() -> Self { let filename = match gen_bool() { true => Some(gen_string()), @@ -79,6 +120,7 @@ impl FileEntry { }; FileEntry { + content_type: rand_mime(), filename: filename, data: PrintHex(gen_bytes()) } @@ -89,6 +131,7 @@ impl FileEntry { } } +#[derive(Hash, PartialEq, Eq)] struct PrintHex(Vec); impl fmt::Debug for PrintHex { @@ -118,7 +161,7 @@ macro_rules! do_test ( info!("Client Test: {:?} Server Test: {:?}", stringify!($client_test), stringify!($server_test)); - let mut test_fields = gen_test_fields(); + let mut test_fields = TestFields::gen(); trace!("Fields for test: {:?}", test_fields); @@ -155,39 +198,11 @@ fn lazy_client_entry_server() { do_test!(test_client_lazy, test_server_entry_api); } -fn gen_test_fields() -> TestFields { - const MIN_FIELDS: usize = 1; - const MAX_FIELDS: usize = 3; - - let mut rng = rand::weak_rng(); - - let texts_count = rng.gen_range(MIN_FIELDS, MAX_FIELDS); - let files_count = rng.gen_range(MIN_FIELDS, MAX_FIELDS); - let nested_count = rng.gen_range(MIN_FIELDS, MAX_FIELDS); - - TestFields { - texts: (0..texts_count).map(|_| (gen_string(), gen_string())).collect(), - files: (0..files_count).map(|_| (gen_string(), FileEntry::gen())).collect(), - nested: (0..nested_count).map(|_| { - let files_count = rng.gen_range(MIN_FIELDS, MAX_FIELDS); - - ( - gen_string(), - (0..files_count).map(|_| FileEntry::gen()).collect() - ) - }).collect() - } -} - fn gen_bool() -> bool { rand::thread_rng().gen() } fn gen_string() -> String { - const MIN_LEN: usize = 2; - const MAX_LEN: usize = 5; - const MAX_DASHES: usize = 2; - let mut rng_1 = rand::thread_rng(); let mut rng_2 = rand::thread_rng(); @@ -210,36 +225,28 @@ fn test_client(test_fields: &TestFields) -> HttpBuffer { let request = ClientRequest::default(); - let mut files = test_fields.files.iter(); - let mut nested_files = test_fields.nested.iter(); + let mut test_files = test_fields.files.iter(); let mut multipart = Multipart::from_request(request).unwrap(); // Intersperse file fields amongst text fields for (name, text) in &test_fields.texts { - if let Some((file_name, file)) = files.next() { - multipart.write_stream(file_name, &mut &*file.data.0, file.filename(), None) - .unwrap(); - } - - if let Some((file_name, files)) = nested_files.next() { - let (data, boundary) = gen_nested_multipart(files); - let mime = format!("multipart/mixed; boundary={}", boundary).parse().unwrap(); - multipart.write_stream(file_name, &mut &*data, None, Some(mime)).unwrap(); + if let Some((file_name, files)) = test_files.next() { + for file in files { + multipart.write_stream(file_name, &mut &*file.data.0, file.filename(), + Some(file.content_type.clone())).unwrap(); + } } multipart.write_text(name, text).unwrap(); } // Write remaining files - for (file_name, file) in files { - multipart.write_stream(file_name, &mut &*file.data.0, None, None).unwrap(); - } - - for (file_name, files) in nested_files { - let (data, boundary) = gen_nested_multipart(files); - let mime = format!("multipart/mixed; boundary={}", boundary).parse().unwrap(); - multipart.write_stream(file_name, &mut &*data, None, Some(mime)).unwrap(); + for (file_name, files) in test_files { + for file in files { + multipart.write_stream(file_name, &mut &*file.data.0, file.filename(), + Some(file.content_type.clone())).unwrap(); + } } multipart.send().unwrap() @@ -251,32 +258,23 @@ fn test_client_lazy(test_fields: &TestFields) -> HttpBuffer { let mut multipart = Multipart::new(); let mut test_files = test_fields.files.iter(); - let mut nested_files = test_fields.nested.iter(); for (name, text) in &test_fields.texts { - for (file_name, file) in &mut test_files { - multipart.add_stream(&**file_name, Cursor::new(&file.data.0), file.filename(), None); - } - - if let Some((file_name, files)) = nested_files.next() { - let (data, boundary) = gen_nested_multipart(files); - let mime = format!("multipart/mixed; boundary={}", boundary).parse().unwrap(); - multipart.add_stream(&**file_name, Cursor::new(data), None as Option<&'static str>, - Some(mime)); + if let Some((file_name, files)) = test_files.next() { + for file in files { + multipart.add_stream(&**file_name, Cursor::new(&file.data.0), file.filename(), + Some(file.content_type.clone())); + } } multipart.add_text(&**name, &**text); } - for (file_name, file) in test_files { - multipart.add_stream(&**file_name, Cursor::new(&file.data.0), None as Option<&str>, None); - } - - for (file_name, files) in nested_files { - let (data, boundary) = gen_nested_multipart(files); - let mime = format!("multipart/mixed; boundary={}", boundary).parse().unwrap(); - multipart.add_stream(&**file_name, Cursor::new(data), None as Option<&'static str>, - Some(mime)); + for (file_name, files) in test_files { + for file in files { + multipart.add_stream(&**file_name, Cursor::new(&file.data.0), file.filename(), + Some(file.content_type.clone())); + } } let mut prepared = multipart.prepare_threshold(None).unwrap(); @@ -329,35 +327,12 @@ fn test_server_entry_api(buf: HttpBuffer, fields: &mut TestFields) { } } -fn gen_nested_multipart(files: &[FileEntry]) -> (Vec, String) { - let mut out = Vec::new(); - let boundary = gen_string(); - - write!(out, "Content-Type: multipart/mixed; boundary={boundary}\r\n\r\n \ - --{boundary}\r\n", boundary=boundary); - - let mut written = false; - - for file in files { - if written { - write!(out, "\r\n--{}\r\n", boundary); - } - - write!(out, "Content-Type: application/octet-stream"); - - if let Some(ref filename) = file.filename { - write!(out, "; filename={}", filename); - } - - write!(out, "\r\n\r\n"); - - out.write_all(&file.data.0); - - written = true; - } - - write!(out, "\r\n--{}--\r\n", boundary); - - (out, boundary) -} - +fn rand_mime() -> Mime { + rand::thread_rng().choose(&[ + // TODO: fill this out, preferably with variants that may be hard to parse + // i.e. containing hyphens, mainly + mime!(Application/OctetStream), + mime!(Text/Plain), + mime!(Image/Png), + ]).unwrap().clone() +} \ No newline at end of file diff --git a/multipart/src/server/boundary.rs b/multipart/src/server/boundary.rs index 4a8710ef8..9276b3886 100644 --- a/multipart/src/server/boundary.rs +++ b/multipart/src/server/boundary.rs @@ -18,8 +18,6 @@ use std::borrow::Borrow; use std::io; use std::io::prelude::*; -use std::mem; - /// A struct implementing `Read` and `BufRead` that will yield bytes until it sees a given sequence. #[derive(Debug)] pub struct BoundaryReader { @@ -166,19 +164,6 @@ impl BoundaryReader where R: Read { Ok(self.at_end) } - - // Keeping this around to support nested boundaries later. - #[allow(unused)] - #[doc(hidden)] - pub fn swap_boundary>>(&mut self, boundary: B) -> Vec { - let mut boundary = boundary.into(); - - if boundary.get(..2) != Some(b"--") { - safemem::prepend(b"--", &mut boundary); - } - - mem::replace(&mut self.boundary, boundary) - } } impl Borrow for BoundaryReader { diff --git a/multipart/src/server/field.rs b/multipart/src/server/field.rs index aac8fb98c..450559037 100644 --- a/multipart/src/server/field.rs +++ b/multipart/src/server/field.rs @@ -9,19 +9,17 @@ use super::httparse::{self, EMPTY_HEADER, Status}; -use super::Multipart; - use self::ReadEntryResult::*; use super::buf_redux::copy_buf; -use mime::{Attr, TopLevel, Mime, Value}; +use mime::{TopLevel, Mime}; use std::fs::{self, OpenOptions}; use std::io::{self, Read, BufRead, Write}; use std::ops::Deref; use std::path::{Path, PathBuf}; -use std::{env, mem, str}; +use std::{env, str}; const RANDOM_FILENAME_LEN: usize = 12; @@ -234,8 +232,6 @@ pub enum MultipartData { Text(MultipartText), /// The field's payload is a binary stream (file). File(MultipartFile), - /// The field's payload is a nested multipart body (multiple files). - Nested(NestedMultipart), } impl MultipartData { @@ -263,7 +259,6 @@ impl MultipartData { match self { Text(text) => text.into_inner(), File(file) => file.into_inner(), - Nested(nested) => nested.into_inner(), } } @@ -273,7 +268,6 @@ impl MultipartData { match *self { Text(ref mut text) => text.take_inner(), File(ref mut file) => file.take_inner(), - Nested(ref mut nested) => nested.take_inner(), } } @@ -285,7 +279,6 @@ impl MultipartData { match *self { Text(ref mut text) => text.inner = inner, File(ref mut file) => file.inner = inner, - Nested(ref mut nested) => nested.inner = inner, } } } @@ -314,10 +307,6 @@ impl Into for MultipartText { } impl MultipartText { - fn inner_mut(&mut self) -> &mut M { - self.inner.as_mut().expect("MultipartText::inner taken!") - } - fn take_inner(&mut self) -> M { self.inner.take().expect("MultipartText::inner taken!") } @@ -333,12 +322,40 @@ impl MultipartText { /// instead, this struct exposes `Read` and `BufRead` impls which point /// to the beginning of the file's contents in the HTTP stream. /// -/// You can read it to EOF, or use one of the `save_*()` methods here +/// You can read it to EOF, or use one of the `save()` method /// to save it to disk. #[derive(Debug)] pub struct MultipartFile { - filename: Option, - content_type: Mime, + /// The filename of this entry, if supplied. + /// + /// ### Warning: Client Provided / Untrustworthy + /// You should treat this value as **untrustworthy** because it is an arbitrary string + /// provided by the client. + /// + /// It is a serious security risk to create files or directories with paths based on user input. + /// A malicious user could craft a path which can be used to overwrite important files, such as + /// web templates, static assets, Javascript files, database files, configuration files, etc., + /// if they are writable by the server process. + /// + /// This can be mitigated somewhat by setting filesystem permissions as + /// conservatively as possible and running the server under its own user with restricted + /// permissions, but you should still not use user input directly as filesystem paths. + /// If it is truly necessary, you should sanitize filenames such that they cannot be + /// misinterpreted by the OS. Such functionality is outside the scope of this crate. + pub filename: Option, + + /// The MIME type (`Content-Type` value) of this file, if supplied by the client, + /// or `"applicaton/octet-stream"` otherwise. + /// + /// ### Note: Client Provided + /// Consider this value to be potentially untrustworthy, as it is provided by the client. + /// It may be inaccurate or entirely wrong, depending on how the client determined it. + /// + /// Some variants wrap arbitrary strings which could be abused by a malicious user if your + /// application performs any non-idempotent operations based on their value, such as + /// starting another program or querying/updating a database (web-search "SQL injection"). + pub content_type: Mime, + /// The `Multipart` this field was read from. inner: Option, } @@ -346,20 +363,41 @@ pub struct MultipartFile { impl MultipartFile { /// Get the filename of this entry, if supplied. /// - /// ##Warning - /// You should treat this value as untrustworthy because it is an arbitrary string provided by - /// the client. You should *not* blindly append it to a directory path and save the file there, - /// as such behavior could easily be exploited by a malicious client. + /// ### Warning: Client Provided / Untrustworthy + /// You should treat this value as **untrustworthy** because it is an arbitrary string + /// provided by the client. + /// + /// It is a serious security risk to create files or directories with paths based on user input. + /// A malicious user could craft a path which can be used to overwrite important files, such as + /// web templates, static assets, Javascript files, database files, configuration files, etc., + /// if they are writable by the server process. + /// + /// This can be mitigated somewhat by setting filesystem permissions as + /// conservatively as possible and running the server under its own user with restricted + /// permissions, but you should still not use user input directly as filesystem paths. + /// If it is truly necessary, you should sanitize filenames such that they cannot be + /// misinterpreted by the OS. Such functionality is outside the scope of this crate. + #[deprecated = "`filename` field is now public"] pub fn filename(&self) -> Option<&str> { self.filename.as_ref().map(String::as_ref) } /// Get the MIME type (`Content-Type` value) of this file, if supplied by the client, /// or `"applicaton/octet-stream"` otherwise. + /// + /// ### Note: Client Provided + /// Consider this value to be potentially untrustworthy, as it is provided by the client. + /// It may be inaccurate or entirely wrong, depending on how the client determined it. + /// + /// Some variants wrap arbitrary strings which could be abused by a malicious user if your + /// application performs any non-idempotent operations based on their value, such as + /// starting another program or querying/updating a database (web-search "SQL injection"). + #[deprecated = "`content_type` field is now public"] pub fn content_type(&self) -> &Mime { &self.content_type } + fn inner_mut(&mut self) -> &mut M { self.inner.as_mut().expect("MultipartFile::inner taken!") } @@ -468,6 +506,7 @@ impl BufRead for MultipartFile { } /// A builder for saving a `MultipartFile` to the local filesystem. +#[must_use = "file is not saved to the filesystem yet"] pub struct SaveBuilder<'m, M: 'm> { file: &'m mut MultipartFile, open_opts: OpenOptions, @@ -571,7 +610,7 @@ impl<'m, M: 'm> SaveBuilder<'m, M> where MultipartFile: BufRead { try!(create_full_path(&path)); - let mut file = try!(self.open_opts.open(&path)); + let file = try!(self.open_opts.open(&path)); let (written, truncated) = try!(self.write_to(file)); Ok(SavedFile { @@ -688,73 +727,6 @@ fn rand_filename() -> String { ::random_alphanumeric(RANDOM_FILENAME_LEN) } -pub struct NestedEntry { - pub content_type: Mime, - pub filename: Option, - inner: M -} - -#[derive(Debug)] -pub struct NestedMultipart { - outer_boundary: Vec, - inner: Option, -} - -impl NestedMultipart { - pub fn read_entry(&mut self) -> ReadEntryResult<&mut M, NestedEntry<&mut M>> - where for<'a> &'a mut M: ReadEntry { - - let inner = self.inner_mut(); - - let headers = match inner.read_headers() { - Ok(Some(headers)) => headers, - Ok(None) => return End(inner), - Err(e) => return Error(inner, e), - }; - - let cont_type = match headers.cont_type { - Some(cont_type) => cont_type, - None => return ReadEntryResult::invalid_data(inner, - "Nested multipart requires Content-Type".to_string()) - }; - - Entry ( - NestedEntry { - filename: headers.cont_disp.filename, - content_type: cont_type, - inner: inner, - } - ) - } - - fn inner_mut(&mut self) -> &mut M { - self.inner.as_mut().expect("NestedMultipart::inner taken!") - } - - fn take_inner(&mut self) -> M { - self.inner.take().expect("NestedMultipart::inner taken!()") - } - - fn into_inner(mut self) -> M { - self.restore_boundary(); - self.inner.take().expect("NestedMultipart::inner taken!()") - } - - fn restore_boundary(&mut self) { - let multipart = self.inner.as_mut().expect("NestedMultipart::inner taken!()"); - let outer_boundary = mem::replace(&mut self.outer_boundary, Vec::new()); - multipart.swap_boundary(outer_boundary); - } -} - -impl Drop for NestedMultipart { - fn drop(&mut self) { - if self.inner.is_some() { - self.restore_boundary(); - } - } -} - /// Common trait for `Multipart` and `&mut Multipart` pub trait ReadEntry: PrivReadEntry {} @@ -770,8 +742,6 @@ pub trait PrivReadEntry: Sized { /// Returns `true` if the last boundary was read, `false` otherwise. fn consume_boundary(&mut self) -> io::Result; - fn swap_boundary>>(&mut self, boundary: B) -> Vec; - fn read_headers(&mut self) -> io::Result> { FieldHeaders::read_from(&mut self.source()) } @@ -798,22 +768,14 @@ pub trait PrivReadEntry: Sized { let data = match field_headers.cont_type { Some(cont_type) => { match cont_type.0 { - TopLevel::Multipart if cont_type.1 == "mixed" => { - let outer_boundary = match cont_type.get_param(Attr::Boundary) { - Some(&Value::Ext(ref boundary)) => self.swap_boundary(&**boundary), - _ => { - let msg = format!("Nested multipart boundary was not provided for \ - field {:?}", field_headers.cont_disp.field_name); - return ReadEntryResult::invalid_data(self, msg); - }, - }; - - MultipartData::Nested( - NestedMultipart { - outer_boundary: outer_boundary, - inner: Some(self), - } - ) + TopLevel::Multipart => { + let msg = format!("Error on field {:?}: nested multipart fields are \ + not supported. However, reports of clients sending \ + requests like this are welcome at \ + https://github.com/abonander/multipart/issues/56", + field_headers.cont_disp.field_name); + + return ReadEntryResult::invalid_data(self, msg); }, _ => { MultipartData::File( @@ -851,9 +813,6 @@ impl<'a, M: ReadEntry> PrivReadEntry for &'a mut M { (**self).source() } - fn swap_boundary>>(&mut self, boundary: B) -> Vec { - (**self).swap_boundary(boundary) - } fn consume_boundary(&mut self) -> io::Result { (**self).consume_boundary() } diff --git a/multipart/src/server/iron.rs b/multipart/src/server/iron.rs index 32f016e06..8c3e80ff6 100644 --- a/multipart/src/server/iron.rs +++ b/multipart/src/server/iron.rs @@ -153,7 +153,7 @@ impl Intercept { file.save().limit(self.file_size_limit).with_dir(&entries.dir); "Error reading field: \"{}\" (filename: \"{}\")", field.name, - file.filename().unwrap_or("(none)") + file.filename.as_ref().map_or("(none)", |f| f) ); if file.size == self.file_size_limit { @@ -173,7 +173,6 @@ impl Intercept { MultipartData::Text(text) => { entries.fields.insert(field.name, text.text); }, - MultipartData::Nested(nested) => unimplemented!(), } } diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index b9e483b57..ef4d694c3 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -219,7 +219,6 @@ impl Multipart { MultipartData::Text(text) => { entries.fields.insert(field.name, text.text); }, - MultipartData::Nested(_) => unimplemented!(), } } @@ -240,10 +239,6 @@ impl PrivReadEntry for Multipart { &mut self.reader } - fn swap_boundary>>(&mut self, boundary: B) -> Vec { - self.reader.swap_boundary(boundary) - } - /// Consume the next boundary. /// Returns `true` if the last boundary was read, `false` otherwise. fn consume_boundary(&mut self) -> io::Result { From 85e8c9b5272b3d064468f2aa25e1fb2b88f26a1d Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Tue, 21 Feb 2017 18:42:19 -0800 Subject: [PATCH 238/453] Bump alpha version --- multipart/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index f9218e758..54b01a3b4 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "multipart" -version = "0.10.0-alpha.2" +version = "0.10.0-alpha.3" authors = ["Austin Bonander "] From f205f3af875e120d4e7b3b4a4e9c6c6920b596b5 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Tue, 21 Feb 2017 19:39:34 -0800 Subject: [PATCH 239/453] Support Iron 0.5 (trivial upgrade) --- multipart/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 54b01a3b4..16784078b 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -30,7 +30,7 @@ twoway = { version = "0.1", optional = true } # Optional Integrations hyper = { version = "0.9", optional = true, default-features = false } -iron = { version = "0.5", optional = true } +iron = { version = ">=0.4,<0.6", optional = true } # NOTE: use `nickel_` feature, as `hyper` feature is required. nickel = { optional = true, version = "0.9" } From e0baf5442f8c847e1f79d119aca32f9fd350ecf7 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Tue, 21 Feb 2017 22:27:41 -0800 Subject: [PATCH 240/453] WIP move saving stuff to new module --- multipart/src/server/field.rs | 202 +------------- multipart/src/server/iron.rs | 3 +- multipart/src/server/mod.rs | 233 ++-------------- multipart/src/server/save.rs | 501 ++++++++++++++++++++++++++++++++++ 4 files changed, 529 insertions(+), 410 deletions(-) create mode 100644 multipart/src/server/save.rs diff --git a/multipart/src/server/field.rs b/multipart/src/server/field.rs index 450559037..c3be05cbb 100644 --- a/multipart/src/server/field.rs +++ b/multipart/src/server/field.rs @@ -11,7 +11,7 @@ use super::httparse::{self, EMPTY_HEADER, Status}; use self::ReadEntryResult::*; -use super::buf_redux::copy_buf; +use super::save::{SaveBuilder, SavedFile}; use mime::{TopLevel, Mime}; @@ -21,8 +21,6 @@ use std::ops::Deref; use std::path::{Path, PathBuf}; use std::{env, str}; -const RANDOM_FILENAME_LEN: usize = 12; - macro_rules! try_io( ($try:expr) => ( { @@ -413,8 +411,8 @@ impl MultipartFile { impl MultipartFile where M: ReadEntry { /// Get a builder type which can save the file with or without a size limit. - pub fn save(&mut self) -> SaveBuilder { - SaveBuilder::from_file(self) + pub fn save(&mut self) -> SaveBuilder { + SaveBuilder::new(self) } /// Save this file to the given output stream. @@ -505,185 +503,6 @@ impl BufRead for MultipartFile { } } -/// A builder for saving a `MultipartFile` to the local filesystem. -#[must_use = "file is not saved to the filesystem yet"] -pub struct SaveBuilder<'m, M: 'm> { - file: &'m mut MultipartFile, - open_opts: OpenOptions, - limit: Option, -} - -impl<'m, M: 'm> SaveBuilder<'m, M> where MultipartFile: BufRead { - fn from_file(file: &'m mut MultipartFile) -> Self { - let mut open_opts = OpenOptions::new(); - open_opts.write(true).create_new(true); - - SaveBuilder { - file: file, - open_opts: open_opts, - limit: None, - } - } - - /// Set the maximum number of bytes to write out. - /// - /// Can be `u64` or `Option`. If `None`, clears the limit. - pub fn limit>>(&mut self, limit: L) -> &mut Self { - self.limit = limit.into(); - self - } - - /// Modify the `OpenOptions` used to open the file for writing. - /// - /// The `write` flag will be reset to `true` after the closure returns. (It'd be pretty - /// pointless otherwise, right?) - pub fn mod_open_opts(&mut self, opts_fn: F) -> &mut Self { - opts_fn(&mut self.open_opts); - self.open_opts.write(true); - self - } - - /// Save to a file with a random alphanumeric name in the given directory. - /// - /// See `with_path()` for more details. - /// - /// ### Warning: Do **not* trust user input! - /// It is a serious security risk to create files or directories with paths based on user input. - /// A malicious user could craft a path which can be used to overwrite important files, such as - /// web templates, static assets, Javascript files, database files, configuration files, etc., - /// if they are writable by the server process. - /// - /// This can be mitigated somewhat by setting filesystem permissions as - /// conservatively as possible and running the server under its own user with restricted - /// permissions, but you should still not use user input directly as filesystem paths. - /// If it is truly necessary, you should sanitize filenames such that they cannot be - /// misinterpreted by the OS. - pub fn with_dir>(&mut self, dir: P) -> io::Result { - let path = dir.as_ref().join(rand_filename()); - self.with_path(path) - } - - /// Save to a file with the given name in the OS temporary directory. - /// - /// See `with_path()` for more details. - /// - /// ### Warning: Do **not* trust user input! - /// It is a serious security risk to create files or directories with paths based on user input. - /// A malicious user could craft a path which can be used to overwrite important files, such as - /// web templates, static assets, Javascript files, database files, configuration files, etc., - /// if they are writable by the server process. - /// - /// This can be mitigated somewhat by setting filesystem permissions as - /// conservatively as possible and running the server under its own user with restricted - /// permissions, but you should still not use user input directly as filesystem paths. - /// If it is truly necessary, you should sanitize filenames such that they cannot be - /// misinterpreted by the OS. - pub fn with_filename>(&mut self, filename: P) -> io::Result { - self.with_path(env::temp_dir().join(filename)) - } - - /// Save to a file with the given path. - /// - /// Truncates the file to the given limit, if set, and the contained `OpenOptions`. - /// - /// ### Warning: Do **not* trust user input! - /// It is a serious security risk to create files or directories with paths based on user input. - /// A malicious user could craft a path which can be used to overwrite important files, such as - /// web templates, static assets, Javascript files, database files, configuration files, etc., - /// if they are writable by the server process. - /// - /// This can be mitigated somewhat by setting filesystem permissions as - /// conservatively as possible and running the server under its own user with restricted - /// permissions, but you should still not use user input directly as filesystem paths. - /// If it is truly necessary, you should sanitize filenames such that they cannot be - /// misinterpreted by the OS. - /// - /// ### `OpenOptions` - /// By default, the open options are set with `.write(true).create_new(true)`, - /// so if the file already exists then an error will be thrown. This is to avoid accidentally - /// overwriting files from other requests. - /// - /// If you want to modify the options used to open the save file, you can use - /// `mod_open_options()`. - pub fn with_path>(&mut self, path: P) -> io::Result { - let path = path.into(); - - try!(create_full_path(&path)); - - let file = try!(self.open_opts.open(&path)); - let (written, truncated) = try!(self.write_to(file)); - - Ok(SavedFile { - path: path, - filename: self.file.filename.clone(), - size: written, - truncated: truncated, - _priv: (), - }) - } - - /// Save to a file with a random alphanumeric name in the OS temporary directory. - /// - /// Does not use user input to create the path. - /// - /// See `with_path()` for more details. - pub fn temp(&mut self) -> io::Result { - let path = env::temp_dir().join(rand_filename()); - self.with_path(path) - } - - /// Write out the file field to `dest`, truncating if a limit was set. - /// - /// Returns the number of bytes copied, and whether or not the limit was reached - /// (tested by `MultipartFile::fill_buf().is_empty()` so no bytes are consumed). - /// - /// Retries on interrupts. - pub fn write_to(&mut self, mut dest: W) -> io::Result<(u64, bool)> { - if let Some(limit) = self.limit { - let copied = try!(copy_buf(&mut self.file.by_ref().take(limit), &mut dest)); - // If there's more data to be read, the field was truncated - Ok((copied, !try!(self.file.fill_buf()).is_empty())) - } else { - copy_buf(&mut self.file, &mut dest).map(|copied| (copied, false)) - } - } -} - -/// A file saved to the local filesystem from a multipart request. -#[derive(Debug)] -pub struct SavedFile { - /// The complete path this file was saved at. - pub path: PathBuf, - - /// The original filename of this file, if one was provided in the request. - /// - /// ##Warning: Provided by Client! Do **not** trust user input! - /// You should treat this value as untrustworthy because it is an arbitrary string provided by - /// the client. You should *not* blindly append it to a directory path and save the file there, - /// as such behavior could easily be exploited by a malicious client. - /// - /// It is a serious security risk to create files or directories with paths based on user input. - /// A malicious user could craft a path which can be used to overwrite important files, such as - /// web templates, static assets, Javascript files, database files, configuration files, etc., - /// if they are writable by the server process. - /// - /// This can be mitigated somewhat by setting filesystem permissions as - /// conservatively as possible and running the server under its own user with restricted - /// permissions, but you should still not use user input directly as filesystem paths. - /// If it is truly necessary, you should sanitize filenames such that they cannot be - /// misinterpreted by the OS. - pub filename: Option, - - /// The number of bytes written to the disk. May be truncated, check the `truncated` flag - /// before making any assumptions based on this number. - pub size: u64, - - /// If the file save limit was hit and the saved file ended up truncated. - pub truncated: bool, - // Private field to prevent exhaustive matching for backwards compatibility - _priv: (), -} - fn read_content_type(cont_type: &str) -> Mime { cont_type.parse().ok().unwrap_or_else(::mime_guess::octet_stream) } @@ -712,21 +531,6 @@ fn find_header<'a, 'b>(headers: &'a [StrHeader<'b>], name: &str) -> Option<&'a S headers.iter().find(|header| header.name == name) } -fn create_full_path(path: &Path) -> io::Result { - if let Some(parent) = path.parent() { - try!(fs::create_dir_all(parent)); - } else { - // RFC: return an error instead? - warn!("Attempting to save file in what looks like a root directory. File path: {:?}", path); - } - - fs::File::create(&path) -} - -fn rand_filename() -> String { - ::random_alphanumeric(RANDOM_FILENAME_LEN) -} - /// Common trait for `Multipart` and `&mut Multipart` pub trait ReadEntry: PrivReadEntry {} diff --git a/multipart/src/server/iron.rs b/multipart/src/server/iron.rs index 8c3e80ff6..9ec080a96 100644 --- a/multipart/src/server/iron.rs +++ b/multipart/src/server/iron.rs @@ -12,7 +12,8 @@ use iron::{BeforeMiddleware, IronError, IronResult}; use std::path::PathBuf; use std::{error, fmt}; -use super::{Entries, HttpRequest, Multipart, MultipartData}; +use super::{HttpRequest, Multipart, MultipartData}; +use super::save::Entries; impl<'r, 'a, 'b> HttpRequest for &'r mut IronRequest<'a, 'b> { type Body = &'r mut IronBody<'a, 'b>; diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index ef4d694c3..34c9e70a5 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -16,8 +16,6 @@ extern crate httparse; extern crate safemem; extern crate twoway; -use tempdir::TempDir; - use std::borrow::Borrow; use std::collections::HashMap; use std::fs; @@ -25,12 +23,17 @@ use std::io::prelude::*; use std::path::{Path, PathBuf}; use std::{io, mem}; +use tempdir::TempDir; + use self::boundary::BoundaryReader; use self::field::PrivReadEntry; -pub use self::field::{MultipartField, MultipartFile, MultipartData, ReadEntry, ReadEntryResult, - SaveBuilder, SavedFile}; +pub use self::field::{MultipartField, MultipartFile, MultipartData, ReadEntry, ReadEntryResult}; + +use self::save::SaveBuilder; + +pub use self::save::{Entries, SaveResult}; macro_rules! try_opt ( ($expr:expr) => ( @@ -74,6 +77,8 @@ pub mod nickel; #[cfg(feature = "tiny_http")] pub mod tiny_http; +pub mod save; + /// The server-side implementation of `multipart/form-data` requests. /// /// Implements `Borrow` to allow access to the request body, if desired. @@ -136,21 +141,18 @@ impl Multipart { } } + pub fn save(&mut self) -> SaveBuilder { + SaveBuilder::new(self) + } + /// Read the request fully, parsing all fields and saving all files in a new temporary /// directory under the OS temporary directory. /// /// If there is an error in reading the request, returns the partial result along with the /// error. See [`SaveResult`](enum.SaveResult.html) for more information. + #[deprecated = "use `.save().temp()` instead"] pub fn save_all(&mut self) -> SaveResult { - let mut entries = match Entries::new_tempdir() { - Ok(entries) => entries, - Err(err) => return SaveResult::Error(err), - }; - - match self.read_to_entries(&mut entries, None) { - Ok(()) => SaveResult::Full(entries), - Err(err) => SaveResult::Partial(entries, err), - } + self.save().temp() } /// Read the request fully, parsing all fields and saving all files in a new temporary @@ -158,15 +160,11 @@ impl Multipart { /// /// If there is an error in reading the request, returns the partial result along with the /// error. See [`SaveResult`](enum.SaveResult.html) for more information. + #[deprecated = "use `.save().with_temp_dir()` instead"] pub fn save_all_under>(&mut self, dir: P) -> SaveResult { - let mut entries = match Entries::new_tempdir_in(dir) { - Ok(entries) => entries, + match TempDir::new_in(dir, "multipart") { + Ok(temp_dir) => self.save().with_temp_dir(temp_dir), Err(err) => return SaveResult::Error(err), - }; - - match self.read_to_entries(&mut entries, None) { - Ok(()) => SaveResult::Full(entries), - Err(err) => SaveResult::Partial(entries, err), } } @@ -177,16 +175,9 @@ impl Multipart { /// /// If there is an error in reading the request, returns the partial result along with the /// error. See [`SaveResult`](enum.SaveResult.html) for more information. + #[deprecated = "use `.save().limit(limit)` instead"] pub fn save_all_limited(&mut self, limit: u64) -> SaveResult { - let mut entries = match Entries::new_tempdir() { - Ok(entries) => entries, - Err(err) => return SaveResult::Error(err), - }; - - match self.read_to_entries(&mut entries, Some(limit)) { - Ok(()) => SaveResult::Full(entries), - Err(err) => SaveResult::Partial(entries, err), - } + self.save().limit(limit).temp() } /// Read the request fully, parsing all fields and saving all files in a new temporary @@ -196,33 +187,12 @@ impl Multipart { /// /// If there is an error in reading the request, returns the partial result along with the /// error. See [`SaveResult`](enum.SaveResult.html) for more information. + #[deprecated = "use `.save().limit(limit).with_temp_dir()` instead"] pub fn save_all_under_limited>(&mut self, dir: P, limit: u64) -> SaveResult { - let mut entries = match Entries::new_tempdir_in(dir) { - Ok(entries) => entries, + match TempDir::new_in(dir, "multipart") { + Ok(temp_dir) => self.save().limit(limit).with_temp_dir(temp_dir), Err(err) => return SaveResult::Error(err), - }; - - match self.read_to_entries(&mut entries, Some(limit)) { - Ok(()) => SaveResult::Full(entries), - Err(err) => SaveResult::Partial(entries, err), - } - } - - fn read_to_entries(&mut self, entries: &mut Entries, limit: Option) -> io::Result<()> { - while let Some(field) = try!(self.read_entry()) { - match field.data { - MultipartData::File(mut file) => { - let file = try!(file.save().limit(limit).with_dir(&entries.dir)); - - entries.files.insert(field.name, file); - }, - MultipartData::Text(text) => { - entries.fields.insert(field.name, text.text); - }, - } } - - Ok(()) } } @@ -247,53 +217,6 @@ impl PrivReadEntry for Multipart { } } -/// The result of [`Multipart::save_all()`](struct.Multipart.html#method.save_all). -#[derive(Debug)] -pub enum SaveResult { - /// The operation was a total success. Contained are all entries of the request. - Full(Entries), - /// The operation errored partway through. Contained are the entries gathered thus far, - /// as well as the error that ended the process. - Partial(Entries, io::Error), - /// The `TempDir` for `Entries` could not be constructed. Contained is the error detailing the - /// problem. - Error(io::Error), -} - -impl SaveResult { - /// Take the `Entries` from `self`, if applicable, and discarding - /// the error, if any. - pub fn to_entries(self) -> Option { - use self::SaveResult::*; - - match self { - Full(entries) | Partial(entries, _) => Some(entries), - Error(_) => None, - } - } - - /// Decompose `self` to `(Option, Option)` - pub fn to_opt(self) -> (Option, Option) { - use self::SaveResult::*; - - match self { - Full(entries) => (Some(entries), None), - Partial(entries, error) => (Some(entries), Some(error)), - Error(error) => (None, Some(error)), - } - } - - /// Map `self` to an `io::Result`, discarding the error in the `Partial` case. - pub fn to_result(self) -> io::Result { - use self::SaveResult::*; - - match self { - Full(entries) | Partial(entries, _) => Ok(entries), - Error(error) => Err(error), - } - } -} - /// A server-side HTTP request that may or may not be multipart. /// /// May be implemented by mutable references if providing the request or body by-value is @@ -311,113 +234,3 @@ pub trait HttpRequest { /// Return the request body for reading. fn body(self) -> Self::Body; } - -/// A result of `Multipart::save_all()`. -#[derive(Debug)] -pub struct Entries { - /// The text fields of the multipart request, mapped by field name -> value. - pub fields: HashMap, - /// A map of file field names to their contents saved on the filesystem. - pub files: HashMap, - /// The directory the files in this request were saved under; may be temporary or permanent. - pub dir: SaveDir, -} - -impl Entries { - fn new_tempdir_in>(path: P) -> io::Result { - TempDir::new_in(path, "multipart").map(Self::with_tempdir) - } - - fn new_tempdir() -> io::Result { - TempDir::new("multipart").map(Self::with_tempdir) - } - - fn with_tempdir(tempdir: TempDir) -> Entries { - Entries { - fields: HashMap::new(), - files: HashMap::new(), - dir: SaveDir::Temp(tempdir), - } - } -} - -/// The save directory for `Entries`. May be temporary (delete-on-drop) or permanent. -#[derive(Debug)] -pub enum SaveDir { - /// This directory is temporary and will be deleted, along with its contents, when this wrapper - /// is dropped. - Temp(TempDir), - /// This directory is permanent and will be left on the filesystem when this wrapper is dropped. - Perm(PathBuf), -} - -impl SaveDir { - /// Get the path of this directory, either temporary or permanent. - pub fn as_path(&self) -> &Path { - use self::SaveDir::*; - match *self { - Temp(ref tempdir) => tempdir.path(), - Perm(ref pathbuf) => &*pathbuf, - } - } - - /// Returns `true` if this is a temporary directory which will be deleted on-drop. - pub fn is_temporary(&self) -> bool { - use self::SaveDir::*; - match *self { - Temp(_) => true, - Perm(_) => false, - } - } - - /// Unwrap the `PathBuf` from `self`; if this is a temporary directory, - /// it will be converted to a permanent one. - pub fn into_path(self) -> PathBuf { - use self::SaveDir::*; - - match self { - Temp(tempdir) => tempdir.into_path(), - Perm(pathbuf) => pathbuf, - } - } - - /// If this `SaveDir` is temporary, convert it to permanent. - /// This is a no-op if it already is permanent. - /// - /// ###Warning: Potential Data Loss - /// Even though this will prevent deletion on-drop, the temporary folder on most OSes - /// (where this directory is created by default) can be automatically cleared by the OS at any - /// time, usually on reboot or when free space is low. - /// - /// It is recommended that you relocate the files from a request which you want to keep to a - /// permanent folder on the filesystem. - pub fn keep(&mut self) { - use self::SaveDir::*; - *self = match mem::replace(self, Perm(PathBuf::new())) { - Temp(tempdir) => Perm(tempdir.into_path()), - old_self => old_self, - }; - } - - /// Delete this directory and its contents, regardless of its permanence. - /// - /// ###Warning: Potential Data Loss - /// This is very likely irreversible, depending on the OS implementation. - /// - /// Files deleted programmatically are deleted directly from disk, as compared to most file - /// manager applications which use a staging area from which deleted files can be safely - /// recovered (i.e. Windows' Recycle Bin, OS X's Trash Can, etc.). - pub fn delete(self) -> io::Result<()> { - use self::SaveDir::*; - match self { - Temp(tempdir) => tempdir.close(), - Perm(pathbuf) => fs::remove_dir_all(&pathbuf), - } - } -} - -impl AsRef for SaveDir { - fn as_ref(&self) -> &Path { - self.as_path() - } -} diff --git a/multipart/src/server/save.rs b/multipart/src/server/save.rs new file mode 100644 index 000000000..e14f1adee --- /dev/null +++ b/multipart/src/server/save.rs @@ -0,0 +1,501 @@ +// Copyright 2016 `multipart` Crate Developers +// +// Licensed under the Apache License, Version 2.0, or the MIT license , at your option. This file may not be +// copied, modified, or distributed except according to those terms. +//! Utilities for saving request entries to the filesystem. + +use super::buf_redux::copy_buf; + +use mime::Mime; + +use super::field::{MultipartData, MultipartFile}; +use super::Multipart; + +pub use tempdir::TempDir; + +use std::collections::HashMap; +use std::io::prelude::*; +use std::fs::OpenOptions; +use std::path::{Path, PathBuf}; +use std::{env, fs, io, mem}; + +const RANDOM_FILENAME_LEN: usize = 12; + +// Because this isn't exposed as a str in the stdlib +#[cfg(not(windows))] +const PATH_SEP: &'static str = "/"; +#[cfg(windows)] +const PATH_SEP: &'static str = "\\"; + +fn rand_filename() -> String { + ::random_alphanumeric(RANDOM_FILENAME_LEN) +} + +/// A builder for saving a file or files to the local filesystem. +/// +/// ### `OpenOptions` +/// This builder holds an instance of `std::fs::OpenOptions` which is used +/// when creating the new file(s). +/// +/// By default, the open options are set with `.write(true).create_new(true)`, +/// so if the file already exists then an error will be thrown. This is to avoid accidentally +/// overwriting files from other requests. +/// +/// If you want to modify the options used to open the save file, you can use +/// `mod_open_opts()`. +/// +/// ### File Size and Count Limits +/// You can set a size limit for individual files with `limit()`, which takes either `u64` +/// or `Option`. +/// +/// You can also set the maximum number of files to process with `count_limit()`, which +/// takes either `u32` or `Option`. This only has an effect when using +/// `SaveBuilder`. +/// +/// ### Warning: Do **not* trust user input! +/// It is a serious security risk to create files or directories with paths based on user input. +/// A malicious user could craft a path which can be used to overwrite important files, such as +/// web templates, static assets, Javascript files, database files, configuration files, etc., +/// if they are writable by the server process. +/// +/// This can be mitigated somewhat by setting filesystem permissions as +/// conservatively as possible and running the server under its own user with restricted +/// permissions, but you should still not use user input directly as filesystem paths. +/// If it is truly necessary, you should sanitize user input such that it cannot cause a path to be +/// misinterpreted by the OS. Such functionality is outside the scope of this crate. +#[must_use = "nothing saved to the filesystem yet"] +pub struct SaveBuilder<'s, S: 's> { + savable: &'s mut S, + open_opts: OpenOptions, + limit: Option, + count_limit: Option, +} + +impl<'s, S: 's> SaveBuilder<'s, S> { + /// Implementation detail but not problematic to have accessible. + #[doc(hidden)] + pub fn new(savable: &'s mut S) -> SaveBuilder<'s, S> { + let mut open_opts = OpenOptions::new(); + open_opts.write(true).create_new(true); + + SaveBuilder { + savable: savable, + open_opts: open_opts, + limit: None, + count_limit: None, + } + } + + /// Set the maximum number of bytes to write out *per file*. + /// + /// Can be `u64` or `Option`. If `None`, clears the limit. + pub fn limit>>(&mut self, limit: L) -> &mut Self { + self.limit = limit.into(); + self + } + + /// Modify the `OpenOptions` used to open any files for writing. + /// + /// The `write` flag will be reset to `true` after the closure returns. (It'd be pretty + /// pointless otherwise, right?) + pub fn mod_open_opts(&mut self, opts_fn: F) -> &mut Self { + opts_fn(&mut self.open_opts); + self.open_opts.write(true); + self + } +} + +impl<'s, R: 's> SaveBuilder<'s, Multipart> where R: Read { + /// Set the maximum number of files to write out. + /// + /// Can be `u32` or `Option`. If `None`, clears the limit. + pub fn count_limit>>(&mut self, count_limit: L) -> &mut Self { + self.count_limit = count_limit.into(); + self + } + + /// Save the file fields in the request to a new temporary directory prefixed with + /// "multipart-rs" in the OS temporary directory. + /// + /// For more options, create a `TempDir` yourself and pass it to `with_temp_dir()` instead. + /// + /// ### Note: Temporary + /// See `SaveDir` for more info (the type of `Entries::save_dir`). + pub fn temp(&mut self) -> SaveResult { + self.temp_with_prefix("multipart-rs") + } + + /// Save the file fields in the request to a new temporary directory with the given string + /// as a prefix in the OS temporary directory. + /// + /// For more options, create a `TempDir` yourself and pass it to `with_temp_dir()` instead. + /// + /// ### Note: Temporary + /// See `SaveDir` for more info (the type of `Entries::save_dir`). + pub fn temp_with_prefix(&mut self, prefix: &str) -> SaveResult { + match TempDir::new(prefix) { + Ok(tempdir) => self.with_temp_dir(tempdir), + Err(e) => SaveResult::Error(e), + } + } + + /// Save the file fields in the request to a new permanent directory with the given path. + /// + /// Any nonexistent parent directories will be created. + pub fn with_dir>(&mut self, dir: P) -> SaveResult { + let dir = dir.into(); + + if let Err(e) = create_dir_all(&dir) { + return SaveResult::Error(e); + }; + + self.with_save_dir(SaveDir::Perm(dir.into())) + } + + /// Save the file fields to the given `TempDir`. + /// + /// The `TempDir` is returned in the result under `Entries::save_dir`. + pub fn with_temp_dir(&mut self, tempdir: TempDir) -> SaveResult { + self.with_save_dir(SaveDir::Temp(tempdir)) + } + + fn with_save_dir(&mut self, save_dir: SaveDir) -> SaveResult { + let mut entries = Entries::new(save_dir); + + let mut count = 0; + + loop { + let field = match self.savable.read_entry() { + Ok(Some(field)) => field, + Ok(None) => break, + Err(e) => return SaveResult::Partial(entries, e), + }; + + match field.data { + MultipartData::File(mut file) => { + match self.count_limit { + Some(limit) if count >= limit => return SaveResult::LimitHit(entries), + _ => (), + } + + match file.save().limit(self.limit).with_dir(&entries.save_dir) { + Ok(saved_file) => entries.files.entry(field.name).or_insert(Vec::new()) + .push(saved_file), + Err(e) => return SaveResult::Partial(entries, e), + } + + count += 1; + }, + MultipartData::Text(text) => { + entries.fields.insert(field.name, text.text); + }, + } + } + + SaveResult::Full(entries) + } +} + +impl<'s, M: 's> SaveBuilder<'s, MultipartFile> where MultipartFile: BufRead { + /// Save to a file with a random alphanumeric name in the given directory. + /// + /// See `with_path()` for more details. + /// + /// ### Warning: Do **not* trust user input! + /// It is a serious security risk to create files or directories with paths based on user input. + /// A malicious user could craft a path which can be used to overwrite important files, such as + /// web templates, static assets, Javascript files, database files, configuration files, etc., + /// if they are writable by the server process. + /// + /// This can be mitigated somewhat by setting filesystem permissions as + /// conservatively as possible and running the server under its own user with restricted + /// permissions, but you should still not use user input directly as filesystem paths. + /// If it is truly necessary, you should sanitize filenames such that they cannot be + /// misinterpreted by the OS. + pub fn with_dir>(&mut self, dir: P) -> io::Result { + let path = dir.as_ref().join(rand_filename()); + self.with_path(path) + } + + /// Save to a file with the given name in the OS temporary directory. + /// + /// See `with_path()` for more details. + /// + /// ### Warning: Do **not* trust user input! + /// It is a serious security risk to create files or directories with paths based on user input. + /// A malicious user could craft a path which can be used to overwrite important files, such as + /// web templates, static assets, Javascript files, database files, configuration files, etc., + /// if they are writable by the server process. + /// + /// This can be mitigated somewhat by setting filesystem permissions as + /// conservatively as possible and running the server under its own user with restricted + /// permissions, but you should still not use user input directly as filesystem paths. + /// If it is truly necessary, you should sanitize filenames such that they cannot be + /// misinterpreted by the OS. + pub fn with_filename(&mut self, filename: &str) -> io::Result { + let mut tempdir = env::temp_dir(); + tempdir.set_file_name(filename); + + self.with_path(tempdir) + } + + /// Save to a file with the given path. + /// + /// Creates any missing directories in the path. + /// Uses the contained `OpenOptions` to create the file. + /// Truncates the file to the given limit, if set. + pub fn with_path>(&mut self, path: P) -> io::Result { + let path = path.into(); + + try!(create_dir_all(&path)); + + let file = try!(self.open_opts.open(&path)); + let (written, truncated) = try!(self.write_to(file)); + + Ok(SavedFile { + path: path, + filename: self.savable.filename.clone(), + content_type: self.savable.content_type.clone(), + size: written, + truncated: truncated, + _priv: (), + }) + } + + /// Save to a file with a random alphanumeric name in the OS temporary directory. + /// + /// Does not use user input to create the path. + /// + /// See `with_path()` for more details. + pub fn temp(&mut self) -> io::Result { + let path = env::temp_dir().join(rand_filename()); + self.with_path(path) + } + + /// Write out the file field to `dest`, truncating if a limit was set. + /// + /// Returns the number of bytes copied, and whether or not the limit was reached + /// (tested by `MultipartFile::fill_buf().is_empty()` so no bytes are consumed). + /// + /// Retries on interrupts. + pub fn write_to(&mut self, mut dest: W) -> io::Result<(u64, bool)> { + if let Some(limit) = self.limit { + let copied = try!(copy_buf(&mut self.savable.by_ref().take(limit), &mut dest)); + // If there's more data to be read, the field was truncated + Ok((copied, !try!(self.savable.fill_buf()).is_empty())) + } else { + copy_buf(&mut self.savable, &mut dest).map(|copied| (copied, false)) + } + } +} + +/// A file saved to the local filesystem from a multipart request. +#[derive(Debug)] +pub struct SavedFile { + /// The complete path this file was saved at. + pub path: PathBuf, + + /// ### Warning: Client Provided / Untrustworthy + /// You should treat this value as **untrustworthy** because it is an arbitrary string + /// provided by the client. + /// + /// It is a serious security risk to create files or directories with paths based on user input. + /// A malicious user could craft a path which can be used to overwrite important files, such as + /// web templates, static assets, Javascript files, database files, configuration files, etc., + /// if they are writable by the server process. + /// + /// This can be mitigated somewhat by setting filesystem permissions as + /// conservatively as possible and running the server under its own user with restricted + /// permissions, but you should still not use user input directly as filesystem paths. + /// If it is truly necessary, you should sanitize filenames such that they cannot be + /// misinterpreted by the OS. Such functionality is outside the scope of this crate. + pub filename: Option, + + /// The MIME type (`Content-Type` value) of this file, if supplied by the client, + /// or `"applicaton/octet-stream"` otherwise. + /// + /// ### Note: Client Provided + /// Consider this value to be potentially untrustworthy, as it is provided by the client. + /// It may be inaccurate or entirely wrong, depending on how the client determined it. + /// + /// Some variants wrap arbitrary strings which could be abused by a malicious user if your + /// application performs any non-idempotent operations based on their value, such as + /// starting another program or querying/updating a database (web-search "SQL injection"). + pub content_type: Mime, + + /// The number of bytes written to the disk. May be truncated, check the `truncated` flag + /// before making any assumptions based on this number. + pub size: u64, + + /// If the file save limit was hit and the saved file ended up truncated. + pub truncated: bool, + // Private field to prevent exhaustive matching for backwards compatibility + _priv: (), +} + +/// A result of `Multipart::save_all()`. +#[derive(Debug)] +pub struct Entries { + /// The text fields of the multipart request, mapped by field name -> value. + pub fields: HashMap, + /// A map of file field names to their contents saved on the filesystem. + pub files: HashMap>, + /// The directory the files in this request were saved under; may be temporary or permanent. + pub save_dir: SaveDir, +} + +impl Entries { + fn new(save_dir: SaveDir) -> Self { + Entries { + fields: HashMap::new(), + files: HashMap::new(), + save_dir: save_dir, + } + } +} + +/// The save directory for `Entries`. May be temporary (delete-on-drop) or permanent. +#[derive(Debug)] +pub enum SaveDir { + /// This directory is temporary and will be deleted, along with its contents, when this wrapper + /// is dropped. + Temp(TempDir), + /// This directory is permanent and will be left on the filesystem when this wrapper is dropped. + Perm(PathBuf), +} + +impl SaveDir { + /// Get the path of this directory, either temporary or permanent. + pub fn as_path(&self) -> &Path { + use self::SaveDir::*; + match *self { + Temp(ref tempdir) => tempdir.path(), + Perm(ref pathbuf) => &*pathbuf, + } + } + + /// Returns `true` if this is a temporary directory which will be deleted on-drop. + pub fn is_temporary(&self) -> bool { + use self::SaveDir::*; + match *self { + Temp(_) => true, + Perm(_) => false, + } + } + + /// Unwrap the `PathBuf` from `self`; if this is a temporary directory, + /// it will be converted to a permanent one. + pub fn into_path(self) -> PathBuf { + use self::SaveDir::*; + + match self { + Temp(tempdir) => tempdir.into_path(), + Perm(pathbuf) => pathbuf, + } + } + + /// If this `SaveDir` is temporary, convert it to permanent. + /// This is a no-op if it already is permanent. + /// + /// ### Warning: Potential Data Loss + /// Even though this will prevent deletion on-drop, the temporary folder on most OSes + /// (where this directory is created by default) can be automatically cleared by the OS at any + /// time, usually on reboot or when free space is low. + /// + /// It is recommended that you relocate the files from a request which you want to keep to a + /// permanent folder on the filesystem. + pub fn keep(&mut self) { + use self::SaveDir::*; + *self = match mem::replace(self, Perm(PathBuf::new())) { + Temp(tempdir) => Perm(tempdir.into_path()), + old_self => old_self, + }; + } + + /// Delete this directory and its contents, regardless of its permanence. + /// + /// ### Warning: Potential Data Loss + /// This is very likely irreversible, depending on the OS implementation. + /// + /// Files deleted programmatically are deleted directly from disk, as compared to most file + /// manager applications which use a staging area from which deleted files can be safely + /// recovered (i.e. Windows' Recycle Bin, OS X's Trash Can, etc.). + pub fn delete(self) -> io::Result<()> { + use self::SaveDir::*; + match self { + Temp(tempdir) => tempdir.close(), + Perm(pathbuf) => fs::remove_dir_all(&pathbuf), + } + } +} + +impl AsRef for SaveDir { + fn as_ref(&self) -> &Path { + self.as_path() + } +} + +/// The result of [`Multipart::save_all()`](struct.multipart.html#method.save_all) +/// and methods on `SaveBuilder`. +#[derive(Debug)] +pub enum SaveResult { + /// The operation was a total success. Contained are all entries of the request. + Full(Entries), + /// The file count limit in the save operation was hit. Contained are the entries + /// that came in under the limit. + LimitHit(Entries), + /// The operation errored partway through. Contained are the entries gathered thus far, + /// as well as the error that ended the process. + Partial(Entries, io::Error), + /// The `TempDir` for `Entries` could not be constructed. Contained is the error detailing the + /// problem. + Error(io::Error), +} + +impl SaveResult { + /// Take the `Entries` from `self`, if applicable, and discarding + /// the error, if any. + pub fn to_entries(self) -> Option { + use self::SaveResult::*; + + match self { + Full(entries) | LimitHit(entries) | Partial(entries, _) => Some(entries), + Error(_) => None, + } + } + + /// Decompose `self` to `(Option, Option)` + pub fn to_opt(self) -> (Option, Option) { + use self::SaveResult::*; + + match self { + Full(entries) => (Some(entries), None), + LimitHit(entries) => (Some(entries), None), + Partial(entries, error) => (Some(entries), Some(error)), + Error(error) => (None, Some(error)), + } + } + + /// Map `self` to an `io::Result`, discarding the error in the `Partial` case. + pub fn to_result(self) -> io::Result { + use self::SaveResult::*; + + match self { + Full(entries) | LimitHit(entries) | Partial(entries, _) => Ok(entries), + Error(error) => Err(error), + } + } +} + +fn create_dir_all(path: &Path) -> io::Result<()> { + if let Some(parent) = path.parent() { + fs::create_dir_all(parent) + } else { + // RFC: return an error instead? + warn!("Attempting to save file in what looks like a root directory. File path: {:?}", path); + + Ok(()) + } +} From 392456f77bbc3e57366df4f7708a2b57bb194b70 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Thu, 23 Feb 2017 14:37:32 -0800 Subject: [PATCH 241/453] WIP changes to save API --- multipart/src/server/save.rs | 211 +++++++++++++++++++++++------------ 1 file changed, 142 insertions(+), 69 deletions(-) diff --git a/multipart/src/server/save.rs b/multipart/src/server/save.rs index e14f1adee..c03382bc2 100644 --- a/multipart/src/server/save.rs +++ b/multipart/src/server/save.rs @@ -13,6 +13,8 @@ use mime::Mime; use super::field::{MultipartData, MultipartFile}; use super::Multipart; +use self::SaveResult::*; + pub use tempdir::TempDir; use std::collections::HashMap; @@ -33,6 +35,24 @@ fn rand_filename() -> String { ::random_alphanumeric(RANDOM_FILENAME_LEN) } +macro_rules! try_start ( + ($try:expr) => ( + match $try { + Ok(val) => val, + Err(e) => return SaveResult::Error(e), + } + ) +); + +macro_rules! try_partial ( + ($try:expr; $partial:expr) => ( + match $try { + Ok(val) => val, + Err(e) => return SaveResult::Partial($partial, e), + } + ) +); + /// A builder for saving a file or files to the local filesystem. /// /// ### `OpenOptions` @@ -123,7 +143,7 @@ impl<'s, R: 's> SaveBuilder<'s, Multipart> where R: Read { /// /// ### Note: Temporary /// See `SaveDir` for more info (the type of `Entries::save_dir`). - pub fn temp(&mut self) -> SaveResult { + pub fn temp(&mut self) -> EntriesSaveResult { self.temp_with_prefix("multipart-rs") } @@ -134,55 +154,58 @@ impl<'s, R: 's> SaveBuilder<'s, Multipart> where R: Read { /// /// ### Note: Temporary /// See `SaveDir` for more info (the type of `Entries::save_dir`). - pub fn temp_with_prefix(&mut self, prefix: &str) -> SaveResult { + pub fn temp_with_prefix(&mut self, prefix: &str) -> EntriesSaveResult { match TempDir::new(prefix) { Ok(tempdir) => self.with_temp_dir(tempdir), Err(e) => SaveResult::Error(e), } } + /// Save the file fields to the given `TempDir`. + /// + /// The `TempDir` is returned in the result under `Entries::save_dir`. + pub fn with_temp_dir(&mut self, tempdir: TempDir) -> EntriesSaveResult { + self.with_entries(Entries::new(SaveDir::Temp(tempdir))) + } + /// Save the file fields in the request to a new permanent directory with the given path. /// /// Any nonexistent parent directories will be created. - pub fn with_dir>(&mut self, dir: P) -> SaveResult { + pub fn with_dir>(&mut self, dir: P) -> EntriesSaveResult { let dir = dir.into(); - if let Err(e) = create_dir_all(&dir) { - return SaveResult::Error(e); - }; + try_start!(create_dir_all(&dir)); - self.with_save_dir(SaveDir::Perm(dir.into())) + self.with_entries(Entries::new(SaveDir::Perm(dir.into()))) } - /// Save the file fields to the given `TempDir`. - /// - /// The `TempDir` is returned in the result under `Entries::save_dir`. - pub fn with_temp_dir(&mut self, tempdir: TempDir) -> SaveResult { - self.with_save_dir(SaveDir::Temp(tempdir)) - } - - fn with_save_dir(&mut self, save_dir: SaveDir) -> SaveResult { - let mut entries = Entries::new(save_dir); - - let mut count = 0; - + pub fn with_entries(&mut self, mut entries: Entries) -> EntriesSaveResult { loop { - let field = match self.savable.read_entry() { - Ok(Some(field)) => field, - Ok(None) => break, - Err(e) => return SaveResult::Partial(entries, e), + let field = match try_partial!(self.savable.read_entry(); entries) { + Some(field) => field, + None => break, }; match field.data { MultipartData::File(mut file) => { match self.count_limit { - Some(limit) if count >= limit => return SaveResult::LimitHit(entries), + Some(limit) if count >= limit => return SaveResult::Partial ( + entries, + LimitHitFile { + field_name: field.name, + filename: file.filename, + content_type: file.content_type, + limit_kind: LimitKind::Count, + } + ), _ => (), } match file.save().limit(self.limit).with_dir(&entries.save_dir) { - Ok(saved_file) => entries.files.entry(field.name).or_insert(Vec::new()) - .push(saved_file), + Ok(saved_file) => if saved_file.truncated { + entries.files.entry(field.name).or_insert(Vec::new()) + .push(saved_file) + }, Err(e) => return SaveResult::Partial(entries, e), } @@ -199,7 +222,19 @@ impl<'s, R: 's> SaveBuilder<'s, Multipart> where R: Read { } impl<'s, M: 's> SaveBuilder<'s, MultipartFile> where MultipartFile: BufRead { - /// Save to a file with a random alphanumeric name in the given directory. + + + /// Save to a file with a random alphanumeric name in the OS temporary directory. + /// + /// Does not use user input to create the path. + /// + /// See `with_path()` for more details. + pub fn temp(&mut self) -> FileSaveResult { + let path = env::temp_dir().join(rand_filename()); + self.with_path(path) + } + + /// Save to a file with the given name in the OS temporary directory. /// /// See `with_path()` for more details. /// @@ -214,12 +249,14 @@ impl<'s, M: 's> SaveBuilder<'s, MultipartFile> where MultipartFile: BufRea /// permissions, but you should still not use user input directly as filesystem paths. /// If it is truly necessary, you should sanitize filenames such that they cannot be /// misinterpreted by the OS. - pub fn with_dir>(&mut self, dir: P) -> io::Result { - let path = dir.as_ref().join(rand_filename()); - self.with_path(path) + pub fn with_filename(&mut self, filename: &str) -> FileSaveResult { + let mut tempdir = env::temp_dir(); + tempdir.set_file_name(filename); + + self.with_path(tempdir) } - /// Save to a file with the given name in the OS temporary directory. + /// Save to a file with a random alphanumeric name in the given directory. /// /// See `with_path()` for more details. /// @@ -234,11 +271,9 @@ impl<'s, M: 's> SaveBuilder<'s, MultipartFile> where MultipartFile: BufRea /// permissions, but you should still not use user input directly as filesystem paths. /// If it is truly necessary, you should sanitize filenames such that they cannot be /// misinterpreted by the OS. - pub fn with_filename(&mut self, filename: &str) -> io::Result { - let mut tempdir = env::temp_dir(); - tempdir.set_file_name(filename); - - self.with_path(tempdir) + pub fn with_dir>(&mut self, dir: P) -> FileSaveResult { + let path = dir.as_ref().join(rand_filename()); + self.with_path(path) } /// Save to a file with the given path. @@ -246,7 +281,7 @@ impl<'s, M: 's> SaveBuilder<'s, MultipartFile> where MultipartFile: BufRea /// Creates any missing directories in the path. /// Uses the contained `OpenOptions` to create the file. /// Truncates the file to the given limit, if set. - pub fn with_path>(&mut self, path: P) -> io::Result { + pub fn with_path>(&mut self, path: P) -> FileSaveResult { let path = path.into(); try!(create_dir_all(&path)); @@ -259,20 +294,9 @@ impl<'s, M: 's> SaveBuilder<'s, MultipartFile> where MultipartFile: BufRea filename: self.savable.filename.clone(), content_type: self.savable.content_type.clone(), size: written, - truncated: truncated, - _priv: (), }) } - /// Save to a file with a random alphanumeric name in the OS temporary directory. - /// - /// Does not use user input to create the path. - /// - /// See `with_path()` for more details. - pub fn temp(&mut self) -> io::Result { - let path = env::temp_dir().join(rand_filename()); - self.with_path(path) - } /// Write out the file field to `dest`, truncating if a limit was set. /// @@ -325,14 +349,8 @@ pub struct SavedFile { /// starting another program or querying/updating a database (web-search "SQL injection"). pub content_type: Mime, - /// The number of bytes written to the disk. May be truncated, check the `truncated` flag - /// before making any assumptions based on this number. + /// The number of bytes written to the disk. pub size: u64, - - /// If the file save limit was hit and the saved file ended up truncated. - pub truncated: bool, - // Private field to prevent exhaustive matching for backwards compatibility - _priv: (), } /// A result of `Multipart::save_all()`. @@ -437,23 +455,78 @@ impl AsRef for SaveDir { } } +/// The file that was to be read next when the limit was hit. +#[derive(Clone, Debug)] +pub struct PartialFile { + /// The field name for this file. + pub field_name: Option, + + /// The path of + pub file_path: Option, + + /// The filename of this entry, if supplied. + /// + /// ### Warning: Client Provided / Untrustworthy + /// You should treat this value as **untrustworthy** because it is an arbitrary string + /// provided by the client. + /// + /// It is a serious security risk to create files or directories with paths based on user input. + /// A malicious user could craft a path which can be used to overwrite important files, such as + /// web templates, static assets, Javascript files, database files, configuration files, etc., + /// if they are writable by the server process. + /// + /// This can be mitigated somewhat by setting filesystem permissions as + /// conservatively as possible and running the server under its own user with restricted + /// permissions, but you should still not use user input directly as filesystem paths. + /// If it is truly necessary, you should sanitize filenames such that they cannot be + /// misinterpreted by the OS. Such functionality is outside the scope of this crate. + pub filename: Option, + + /// The MIME type (`Content-Type` value) of this file, if supplied by the client, + /// or `"applicaton/octet-stream"` otherwise. + /// + /// ### Note: Client Provided + /// Consider this value to be potentially untrustworthy, as it is provided by the client. + /// It may be inaccurate or entirely wrong, depending on how the client determined it. + /// + /// Some variants wrap arbitrary strings which could be abused by a malicious user if your + /// application performs any non-idempotent operations based on their value, such as + /// starting another program or querying/updating a database (web-search "SQL injection"). + pub content_type: Mime, +} + +/// The kind of the limit that was hit +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PartialReason { + CountLimit, + SizeLimit, + IoError(io::Error), +} + +pub struct PartialEntries { + pub entries: Entries, + pub errored_file: Option, +} + /// The result of [`Multipart::save_all()`](struct.multipart.html#method.save_all) /// and methods on `SaveBuilder`. #[derive(Debug)] -pub enum SaveResult { - /// The operation was a total success. Contained are all entries of the request. - Full(Entries), - /// The file count limit in the save operation was hit. Contained are the entries - /// that came in under the limit. - LimitHit(Entries), - /// The operation errored partway through. Contained are the entries gathered thus far, - /// as well as the error that ended the process. - Partial(Entries, io::Error), - /// The `TempDir` for `Entries` could not be constructed. Contained is the error detailing the - /// problem. +pub enum SaveResult { + /// The operation was a total success. Contained is the complete result. + Full(Success), + /// The operation quit partway through. Included is the partial + /// result along with the reason. + Partial(Partial, PartialReason), + /// An error occurred at the start of the operation, before anything was done. Error(io::Error), } +/// Shorthand result for methods that return `Entries` +pub type EntriesSaveResult = SaveResult; + +/// Shorthand result for methods that return `SavedFile`s. +pub type FileSaveResult = SaveResult; + impl SaveResult { /// Take the `Entries` from `self`, if applicable, and discarding /// the error, if any. @@ -461,7 +534,7 @@ impl SaveResult { use self::SaveResult::*; match self { - Full(entries) | LimitHit(entries) | Partial(entries, _) => Some(entries), + Full(entries) | LimitHit(entries, _) | Partial(entries, _) => Some(entries), Error(_) => None, } } @@ -472,7 +545,7 @@ impl SaveResult { match self { Full(entries) => (Some(entries), None), - LimitHit(entries) => (Some(entries), None), + LimitHit(entries, _) => (Some(entries), None), Partial(entries, error) => (Some(entries), Some(error)), Error(error) => (None, Some(error)), } @@ -483,7 +556,7 @@ impl SaveResult { use self::SaveResult::*; match self { - Full(entries) | LimitHit(entries) | Partial(entries, _) => Ok(entries), + Full(entries) | LimitHit(entries, _) | Partial(entries, _) => Ok(entries), Error(error) => Err(error), } } From d7d5759cd566d4259af59a76079e166e697f95ac Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Fri, 24 Feb 2017 01:35:26 -0800 Subject: [PATCH 242/453] WIP save API --- multipart/src/server/mod.rs | 10 +++++--- multipart/src/server/save.rs | 49 ++++++++++++++++++++++++++++-------- 2 files changed, 44 insertions(+), 15 deletions(-) diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index 34c9e70a5..11051e099 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -35,6 +35,8 @@ use self::save::SaveBuilder; pub use self::save::{Entries, SaveResult}; +use self::save::EntriesSaveResult; + macro_rules! try_opt ( ($expr:expr) => ( match $expr { @@ -151,7 +153,7 @@ impl Multipart { /// If there is an error in reading the request, returns the partial result along with the /// error. See [`SaveResult`](enum.SaveResult.html) for more information. #[deprecated = "use `.save().temp()` instead"] - pub fn save_all(&mut self) -> SaveResult { + pub fn save_all(&mut self) -> EntriesSaveResult { self.save().temp() } @@ -161,7 +163,7 @@ impl Multipart { /// If there is an error in reading the request, returns the partial result along with the /// error. See [`SaveResult`](enum.SaveResult.html) for more information. #[deprecated = "use `.save().with_temp_dir()` instead"] - pub fn save_all_under>(&mut self, dir: P) -> SaveResult { + pub fn save_all_under>(&mut self, dir: P) -> EntriesSaveResult { match TempDir::new_in(dir, "multipart") { Ok(temp_dir) => self.save().with_temp_dir(temp_dir), Err(err) => return SaveResult::Error(err), @@ -176,7 +178,7 @@ impl Multipart { /// If there is an error in reading the request, returns the partial result along with the /// error. See [`SaveResult`](enum.SaveResult.html) for more information. #[deprecated = "use `.save().limit(limit)` instead"] - pub fn save_all_limited(&mut self, limit: u64) -> SaveResult { + pub fn save_all_limited(&mut self, limit: u64) -> EntriesSaveResult { self.save().limit(limit).temp() } @@ -188,7 +190,7 @@ impl Multipart { /// If there is an error in reading the request, returns the partial result along with the /// error. See [`SaveResult`](enum.SaveResult.html) for more information. #[deprecated = "use `.save().limit(limit).with_temp_dir()` instead"] - pub fn save_all_under_limited>(&mut self, dir: P, limit: u64) -> SaveResult { + pub fn save_all_under_limited>(&mut self, dir: P, limit: u64) -> EntriesSaveResult { match TempDir::new_in(dir, "multipart") { Ok(temp_dir) => self.save().limit(limit).with_temp_dir(temp_dir), Err(err) => return SaveResult::Error(err), diff --git a/multipart/src/server/save.rs b/multipart/src/server/save.rs index c03382bc2..651b7c435 100644 --- a/multipart/src/server/save.rs +++ b/multipart/src/server/save.rs @@ -180,6 +180,8 @@ impl<'s, R: 's> SaveBuilder<'s, Multipart> where R: Read { } pub fn with_entries(&mut self, mut entries: Entries) -> EntriesSaveResult { + let mut count = 0; + loop { let field = match try_partial!(self.savable.read_entry(); entries) { Some(field) => field, @@ -191,22 +193,33 @@ impl<'s, R: 's> SaveBuilder<'s, Multipart> where R: Read { match self.count_limit { Some(limit) if count >= limit => return SaveResult::Partial ( entries, - LimitHitFile { - field_name: field.name, + PartialFile { + field_name: Some(field.name), + file_path: None, filename: file.filename, content_type: file.content_type, - limit_kind: LimitKind::Count, + written: 0, } ), _ => (), } match file.save().limit(self.limit).with_dir(&entries.save_dir) { - Ok(saved_file) => if saved_file.truncated { - entries.files.entry(field.name).or_insert(Vec::new()) - .push(saved_file) - }, - Err(e) => return SaveResult::Partial(entries, e), + Success(saved_file) => entries.mut_files_for(&field.name).push(saved_file), + Partial(partial, reason) => return Partial( + PartialEntries { + entries: entries, + errored_file: Some(partial), + }, + reason + ), + Err(e) => return Partial( + PartialEntries { + entries: entries, + errored_file: Some(partial), + }, + PartialReason::IoError(error) + ), } count += 1; @@ -223,7 +236,6 @@ impl<'s, R: 's> SaveBuilder<'s, Multipart> where R: Read { impl<'s, M: 's> SaveBuilder<'s, MultipartFile> where MultipartFile: BufRead { - /// Save to a file with a random alphanumeric name in the OS temporary directory. /// /// Does not use user input to create the path. @@ -372,6 +384,10 @@ impl Entries { save_dir: save_dir, } } + + pub fn mut_files_for(&mut self, field: &str) -> &mut Vec { + self.files.entry(field).or_insert_with(Vec::new) + } } /// The save directory for `Entries`. May be temporary (delete-on-drop) or permanent. @@ -458,10 +474,10 @@ impl AsRef for SaveDir { /// The file that was to be read next when the limit was hit. #[derive(Clone, Debug)] pub struct PartialFile { - /// The field name for this file. + /// The field name for this file, if available. pub field_name: Option, - /// The path of + /// The path of the file, if one was already created. pub file_path: Option, /// The filename of this entry, if supplied. @@ -493,6 +509,17 @@ pub struct PartialFile { /// application performs any non-idempotent operations based on their value, such as /// starting another program or querying/updating a database (web-search "SQL injection"). pub content_type: Mime, + + /// The number of bytes that were written out before the error occurred. + pub written: u64, + __priv: (), +} + +impl PartialFile { + fn with_field_name(mut self, field_name: String) -> Self { + self.field_name = Some(field_name); + self + } } /// The kind of the limit that was hit From b1e0f10b8d924e8b4d82f86faec3892d16c76af6 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Fri, 24 Feb 2017 14:11:21 -0800 Subject: [PATCH 243/453] WIP save API (almost compiles) --- multipart/src/server/field.rs | 19 ++- multipart/src/server/mod.rs | 16 +-- multipart/src/server/save.rs | 243 +++++++++++++++++++++------------- 3 files changed, 171 insertions(+), 107 deletions(-) diff --git a/multipart/src/server/field.rs b/multipart/src/server/field.rs index c3be05cbb..01bc7bd7e 100644 --- a/multipart/src/server/field.rs +++ b/multipart/src/server/field.rs @@ -11,7 +11,8 @@ use super::httparse::{self, EMPTY_HEADER, Status}; use self::ReadEntryResult::*; -use super::save::{SaveBuilder, SavedFile}; +use super::save::{PartialReason, SaveBuilder, SavedFile}; +use super::save::SaveResult::*; use mime::{TopLevel, Mime}; @@ -225,7 +226,7 @@ impl MultipartField { /// The data of a field in a `multipart/form-data` request. #[derive(Debug)] -pub enum MultipartData { +pub enum MultipartData { /// The field's payload is a text string. Text(MultipartText), /// The field's payload is a binary stream (file). @@ -422,7 +423,12 @@ impl MultipartFile where M: ReadEntry { /// Retries when `io::Error::kind() == io::ErrorKind::Interrupted`. #[deprecated = "use `.save().write_to()` instead"] pub fn save_to(&mut self, out: W) -> io::Result { - self.save().write_to(out).map(|(size, _)| size) + match self.save().write_to(out) { + Full(copied) => Ok(copied), + Partial(copied, PartialReason::IoError(e)) => Err(e), + Partial(_, _) => unreachable!(), + Error(e) => Err(e), + } } /// Save this file to the given output stream, **truncated** to `limit` @@ -433,7 +439,12 @@ impl MultipartFile where M: ReadEntry { /// Retries when `io::Error::kind() == io::ErrorKind::Interrupted`. #[deprecated = "use `.save().limit(limit).write_to(out)` instead"] pub fn save_to_limited(&mut self, out: W, limit: u64) -> io::Result { - self.save().limit(limit).write_to(out).map(|(size, _)| size) + match self.save().limit(limit).write_to(out) { + Full(copied) => Ok(copied), + Partial(copied, PartialReason::IoError(e)) => Err(e), + Partial(copied, _) => Ok(copied), + Error(e) => Err(e), + } } /// Save this file to `path`. diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index 11051e099..b9abc3637 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -84,8 +84,8 @@ pub mod save; /// The server-side implementation of `multipart/form-data` requests. /// /// Implements `Borrow` to allow access to the request body, if desired. -pub struct Multipart { - reader: BoundaryReader, +pub struct Multipart { + reader: BoundaryReader, } impl Multipart<()> { @@ -103,9 +103,9 @@ impl Multipart<()> { } } -impl Multipart { +impl Multipart { /// Construct a new `Multipart` with the given body reader and boundary. - pub fn with_body>(body: B, boundary: Bnd) -> Self { + pub fn with_body>(body: R, boundary: Bnd) -> Self { Multipart { reader: BoundaryReader::from_reader(body, boundary.into()), } @@ -153,7 +153,7 @@ impl Multipart { /// If there is an error in reading the request, returns the partial result along with the /// error. See [`SaveResult`](enum.SaveResult.html) for more information. #[deprecated = "use `.save().temp()` instead"] - pub fn save_all(&mut self) -> EntriesSaveResult { + pub fn save_all(&mut self) -> EntriesSaveResult { self.save().temp() } @@ -163,7 +163,7 @@ impl Multipart { /// If there is an error in reading the request, returns the partial result along with the /// error. See [`SaveResult`](enum.SaveResult.html) for more information. #[deprecated = "use `.save().with_temp_dir()` instead"] - pub fn save_all_under>(&mut self, dir: P) -> EntriesSaveResult { + pub fn save_all_under>(&mut self, dir: P) -> EntriesSaveResult { match TempDir::new_in(dir, "multipart") { Ok(temp_dir) => self.save().with_temp_dir(temp_dir), Err(err) => return SaveResult::Error(err), @@ -178,7 +178,7 @@ impl Multipart { /// If there is an error in reading the request, returns the partial result along with the /// error. See [`SaveResult`](enum.SaveResult.html) for more information. #[deprecated = "use `.save().limit(limit)` instead"] - pub fn save_all_limited(&mut self, limit: u64) -> EntriesSaveResult { + pub fn save_all_limited(&mut self, limit: u64) -> EntriesSaveResult { self.save().limit(limit).temp() } @@ -190,7 +190,7 @@ impl Multipart { /// If there is an error in reading the request, returns the partial result along with the /// error. See [`SaveResult`](enum.SaveResult.html) for more information. #[deprecated = "use `.save().limit(limit).with_temp_dir()` instead"] - pub fn save_all_under_limited>(&mut self, dir: P, limit: u64) -> EntriesSaveResult { + pub fn save_all_under_limited>(&mut self, dir: P, limit: u64) -> EntriesSaveResult { match TempDir::new_in(dir, "multipart") { Ok(temp_dir) => self.save().limit(limit).with_temp_dir(temp_dir), Err(err) => return SaveResult::Error(err), diff --git a/multipart/src/server/save.rs b/multipart/src/server/save.rs index 651b7c435..c50d11d13 100644 --- a/multipart/src/server/save.rs +++ b/multipart/src/server/save.rs @@ -48,7 +48,7 @@ macro_rules! try_partial ( ($try:expr; $partial:expr) => ( match $try { Ok(val) => val, - Err(e) => return SaveResult::Partial($partial, e), + Err(e) => Partial($partial, e.into()), } ) ); @@ -143,7 +143,7 @@ impl<'s, R: 's> SaveBuilder<'s, Multipart> where R: Read { /// /// ### Note: Temporary /// See `SaveDir` for more info (the type of `Entries::save_dir`). - pub fn temp(&mut self) -> EntriesSaveResult { + pub fn temp(&mut self) -> EntriesSaveResult<'s, R> { self.temp_with_prefix("multipart-rs") } @@ -154,7 +154,7 @@ impl<'s, R: 's> SaveBuilder<'s, Multipart> where R: Read { /// /// ### Note: Temporary /// See `SaveDir` for more info (the type of `Entries::save_dir`). - pub fn temp_with_prefix(&mut self, prefix: &str) -> EntriesSaveResult { + pub fn temp_with_prefix(&mut self, prefix: &str) -> EntriesSaveResult<'s, R> { match TempDir::new(prefix) { Ok(tempdir) => self.with_temp_dir(tempdir), Err(e) => SaveResult::Error(e), @@ -164,14 +164,14 @@ impl<'s, R: 's> SaveBuilder<'s, Multipart> where R: Read { /// Save the file fields to the given `TempDir`. /// /// The `TempDir` is returned in the result under `Entries::save_dir`. - pub fn with_temp_dir(&mut self, tempdir: TempDir) -> EntriesSaveResult { + pub fn with_temp_dir(&mut self, tempdir: TempDir) -> EntriesSaveResult<'s, R> { self.with_entries(Entries::new(SaveDir::Temp(tempdir))) } /// Save the file fields in the request to a new permanent directory with the given path. /// /// Any nonexistent parent directories will be created. - pub fn with_dir>(&mut self, dir: P) -> EntriesSaveResult { + pub fn with_dir>(&mut self, dir: P) -> EntriesSaveResult<'s, R> { let dir = dir.into(); try_start!(create_dir_all(&dir)); @@ -179,11 +179,14 @@ impl<'s, R: 's> SaveBuilder<'s, Multipart> where R: Read { self.with_entries(Entries::new(SaveDir::Perm(dir.into()))) } - pub fn with_entries(&mut self, mut entries: Entries) -> EntriesSaveResult { + pub fn with_entries(&mut self, mut entries: Entries) -> EntriesSaveResult<'s, R> { let mut count = 0; loop { - let field = match try_partial!(self.savable.read_entry(); entries) { + let field = match try_partial!(self.savable.read_entry(); PartialEntries { + entries: entries, + partial_field: None, + }) { Some(field) => field, None => break, }; @@ -192,37 +195,42 @@ impl<'s, R: 's> SaveBuilder<'s, Multipart> where R: Read { MultipartData::File(mut file) => { match self.count_limit { Some(limit) if count >= limit => return SaveResult::Partial ( - entries, - PartialFile { - field_name: Some(field.name), - file_path: None, - filename: file.filename, - content_type: file.content_type, - written: 0, - } + PartialEntries { + entries: entries, + partial_field: Some(PartialFileField { + field_name: field.name, + file: PartialFile::just_file(file) + }) + }, + PartialReason::CountLimit, ), _ => (), } + count += 1; + match file.save().limit(self.limit).with_dir(&entries.save_dir) { - Success(saved_file) => entries.mut_files_for(&field.name).push(saved_file), + Full(saved_file) => entries.mut_files_for(&field.name).push(saved_file), Partial(partial, reason) => return Partial( PartialEntries { entries: entries, - errored_file: Some(partial), - }, - reason + partial_field: Some(PartialFileField { + field_name: field.name, + file: partial.with_file(file), + }) + } ), Err(e) => return Partial( PartialEntries { entries: entries, - errored_file: Some(partial), + partial_field: Some(PartialFileField { + field_name: field.name, + file: PartialFile::just_file(file) + }), }, - PartialReason::IoError(error) + e.into(), ), } - - count += 1; }, MultipartData::Text(text) => { entries.fields.insert(field.name, text.text); @@ -296,17 +304,20 @@ impl<'s, M: 's> SaveBuilder<'s, MultipartFile> where MultipartFile: BufRea pub fn with_path>(&mut self, path: P) -> FileSaveResult { let path = path.into(); - try!(create_dir_all(&path)); - - let file = try!(self.open_opts.open(&path)); - let (written, truncated) = try!(self.write_to(file)); + let file = match create_dir_all(&path).and_then(|_| self.open_opts.open(&path)) { + Ok(file) => file, + Err(e) => return Partial( + PartialFile { + file_path: Some(path), + file: (), + written: 0, + __priv: (), + }, + e.into(), + ) + }; - Ok(SavedFile { - path: path, - filename: self.savable.filename.clone(), - content_type: self.savable.content_type.clone(), - size: written, - }) + self.write_to(file) } @@ -316,13 +327,21 @@ impl<'s, M: 's> SaveBuilder<'s, MultipartFile> where MultipartFile: BufRea /// (tested by `MultipartFile::fill_buf().is_empty()` so no bytes are consumed). /// /// Retries on interrupts. - pub fn write_to(&mut self, mut dest: W) -> io::Result<(u64, bool)> { + pub fn write_to(&mut self, mut dest: W) -> SaveResult { if let Some(limit) = self.limit { - let copied = try!(copy_buf(&mut self.savable.by_ref().take(limit), &mut dest)); + let copied = match try_copy_buf(self.savable.take(limit), &mut dest) { + Full(copied) => copied, + other => return other, + }; + // If there's more data to be read, the field was truncated - Ok((copied, !try!(self.savable.fill_buf()).is_empty())) + match self.savable.fill_buf() { + Ok(ref buf) if buf.is_empty() => Full(copied), + Ok(_) => Partial(copied, PartialReason::SizeLimit), + Err(e) => Partial(copied, PartialReason::IoError((e))) + } } else { - copy_buf(&mut self.savable, &mut dest).map(|copied| (copied, false)) + try_copy_buf(&mut self.savable, &mut dest) } } } @@ -473,66 +492,72 @@ impl AsRef for SaveDir { /// The file that was to be read next when the limit was hit. #[derive(Clone, Debug)] -pub struct PartialFile { - /// The field name for this file, if available. - pub field_name: Option, - - /// The path of the file, if one was already created. - pub file_path: Option, - - /// The filename of this entry, if supplied. +pub struct PartialFile { + /// The path of the file on the filesystem. /// - /// ### Warning: Client Provided / Untrustworthy - /// You should treat this value as **untrustworthy** because it is an arbitrary string - /// provided by the client. - /// - /// It is a serious security risk to create files or directories with paths based on user input. - /// A malicious user could craft a path which can be used to overwrite important files, such as - /// web templates, static assets, Javascript files, database files, configuration files, etc., - /// if they are writable by the server process. - /// - /// This can be mitigated somewhat by setting filesystem permissions as - /// conservatively as possible and running the server under its own user with restricted - /// permissions, but you should still not use user input directly as filesystem paths. - /// If it is truly necessary, you should sanitize filenames such that they cannot be - /// misinterpreted by the OS. Such functionality is outside the scope of this crate. - pub filename: Option, + /// If an error occurred while creating the file, this will not exist. + pub file_path: Option, - /// The MIME type (`Content-Type` value) of this file, if supplied by the client, - /// or `"applicaton/octet-stream"` otherwise. - /// - /// ### Note: Client Provided - /// Consider this value to be potentially untrustworthy, as it is provided by the client. - /// It may be inaccurate or entirely wrong, depending on how the client determined it. - /// - /// Some variants wrap arbitrary strings which could be abused by a malicious user if your - /// application performs any non-idempotent operations based on their value, such as - /// starting another program or querying/updating a database (web-search "SQL injection"). - pub content_type: Mime, + /// The file in the multipart stream. + pub file: F, - /// The number of bytes that were written out before the error occurred. + /// The number of bytes written to the filesystem. pub written: u64, + __priv: (), } -impl PartialFile { - fn with_field_name(mut self, field_name: String) -> Self { - self.field_name = Some(field_name); - self +impl PartialFile { + fn with_file(self, file: F_) -> PartialFile { + PartialFile { + file_path: self.file_path, + file: file, + written: self.written, + __priv: () + } + } + + fn just_file(file: F) -> Self { + PartialFile { + file_path: None, + file: file, + written: 0, + __priv: () + } } } -/// The kind of the limit that was hit -#[derive(Debug, Clone, PartialEq, Eq)] +/// The reason the save operation quit partway through. +#[derive(Debug, Clone)] pub enum PartialReason { + /// The count limit for files in the request was hit. + /// + /// The associated file has not been saved to the filesystem. CountLimit, + /// The size limit for an individual file was hit. + /// + /// The file was partially written to the filesystem. SizeLimit, + /// An error occurred during the operation. IoError(io::Error), } -pub struct PartialEntries { +impl From for PartialReason { + fn from(e: io::Error) -> Self { + PartialReason::IoError(e) + } +} + +pub struct PartialFileField<'m, R: 'm> { + /// The field name for the errored file. + pub field_name: String, + pub file: PartialFile>>, + +} + +pub struct PartialEntries<'m, R: 'm> { pub entries: Entries, - pub errored_file: Option, + pub partial_field: Option>, } /// The result of [`Multipart::save_all()`](struct.multipart.html#method.save_all) @@ -549,41 +574,37 @@ pub enum SaveResult { } /// Shorthand result for methods that return `Entries` -pub type EntriesSaveResult = SaveResult; +pub type EntriesSaveResult<'m, R> = SaveResult>; /// Shorthand result for methods that return `SavedFile`s. -pub type FileSaveResult = SaveResult; +/// +/// The `MultipartFile` is not available here because it is not necessary to return +/// a borrow when the owned version is probably in the same scope. +pub type FileSaveResult = SaveResult>; -impl SaveResult { +impl<'m, R> EntriesSaveResult<'m, R> { /// Take the `Entries` from `self`, if applicable, and discarding /// the error, if any. pub fn to_entries(self) -> Option { - use self::SaveResult::*; - match self { - Full(entries) | LimitHit(entries, _) | Partial(entries, _) => Some(entries), + Full(entries) | Partial(PartialEntries { entries, .. }, _) => Some(entries), Error(_) => None, } } /// Decompose `self` to `(Option, Option)` pub fn to_opt(self) -> (Option, Option) { - use self::SaveResult::*; - match self { - Full(entries) => (Some(entries), None), - LimitHit(entries, _) => (Some(entries), None), - Partial(entries, error) => (Some(entries), Some(error)), + Partial(PartialEntries { entries, .. }, PartialReason::IoError(e)) => (Some(entries), Some(e)), + Full(entries) | Partial(entries, _) => (Some(entries), None), Error(error) => (None, Some(error)), } } /// Map `self` to an `io::Result`, discarding the error in the `Partial` case. pub fn to_result(self) -> io::Result { - use self::SaveResult::*; - match self { - Full(entries) | LimitHit(entries, _) | Partial(entries, _) => Ok(entries), + Full(entries) | Partial(PartialEntries { entries, .. }, _) => Ok(entries), Error(error) => Err(error), } } @@ -595,7 +616,39 @@ fn create_dir_all(path: &Path) -> io::Result<()> { } else { // RFC: return an error instead? warn!("Attempting to save file in what looks like a root directory. File path: {:?}", path); - Ok(()) } } + +fn try_copy_buf(mut src: R, dest: W) -> SaveResult { + let mut total_copied = 0; + + macro_rules! try_here ( + ($try:expr) => ( + match $try { + Ok(val) => val, + Err(ref e) if e.kind() == io::ErrorKind::Interrupted => continue, + Err(e) => return if total_copied == 0 { Error(e) } + else { Partial(total_copied, e.into()) }, + } + ) + ); + + loop { + let mut buf = try_here!(src.fill_buf()); + + while !buf.is_empty() { + match try_here!(dest.write(buf)) { + 0 => try_here!(Err(io::Error::new(io::ErrorKind::WriteZero, + "failed to write whole buffer"))), + copied => { + buf = &mut buf[copied..]; + total_copied += copied as u64; + src.consume(copied) + }, + } + } + } + + Full(total_copied) +} From 4b78fe4b75b1b8905f2663f73fdc4efc2a825c78 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Fri, 24 Feb 2017 19:21:19 -0800 Subject: [PATCH 244/453] WIP saving, just need to update `iron` mod --- multipart/src/server/field.rs | 79 ++++++------- multipart/src/server/mod.rs | 4 +- multipart/src/server/save.rs | 216 ++++++++++++++++++---------------- 3 files changed, 152 insertions(+), 147 deletions(-) diff --git a/multipart/src/server/field.rs b/multipart/src/server/field.rs index 01bc7bd7e..86083fb71 100644 --- a/multipart/src/server/field.rs +++ b/multipart/src/server/field.rs @@ -12,7 +12,6 @@ use super::httparse::{self, EMPTY_HEADER, Status}; use self::ReadEntryResult::*; use super::save::{PartialReason, SaveBuilder, SavedFile}; -use super::save::SaveResult::*; use mime::{TopLevel, Mime}; @@ -412,7 +411,7 @@ impl MultipartFile { impl MultipartFile where M: ReadEntry { /// Get a builder type which can save the file with or without a size limit. - pub fn save(&mut self) -> SaveBuilder { + pub fn save(&mut self) -> SaveBuilder<&mut Self> { SaveBuilder::new(self) } @@ -423,12 +422,7 @@ impl MultipartFile where M: ReadEntry { /// Retries when `io::Error::kind() == io::ErrorKind::Interrupted`. #[deprecated = "use `.save().write_to()` instead"] pub fn save_to(&mut self, out: W) -> io::Result { - match self.save().write_to(out) { - Full(copied) => Ok(copied), - Partial(copied, PartialReason::IoError(e)) => Err(e), - Partial(_, _) => unreachable!(), - Error(e) => Err(e), - } + self.save().write_to(out).into_result_strict() } /// Save this file to the given output stream, **truncated** to `limit` @@ -439,12 +433,7 @@ impl MultipartFile where M: ReadEntry { /// Retries when `io::Error::kind() == io::ErrorKind::Interrupted`. #[deprecated = "use `.save().limit(limit).write_to(out)` instead"] pub fn save_to_limited(&mut self, out: W, limit: u64) -> io::Result { - match self.save().limit(limit).write_to(out) { - Full(copied) => Ok(copied), - Partial(copied, PartialReason::IoError(e)) => Err(e), - Partial(copied, _) => Ok(copied), - Error(e) => Err(e), - } + self.save().limit(limit).write_to(out).into_result_strict() } /// Save this file to `path`. @@ -454,7 +443,7 @@ impl MultipartFile where M: ReadEntry { /// Retries when `io::Error::kind() == io::ErrorKind::Interrupted`. #[deprecated = "use `.save().with_path(path)` instead"] pub fn save_as>(&mut self, path: P) -> io::Result { - self.save().with_path(path) + self.save().with_path(path).into_result_strict() } /// Save this file in the directory pointed at by `dir`, @@ -467,7 +456,7 @@ impl MultipartFile where M: ReadEntry { /// Retries when `io::Error::kind() == io::ErrorKind::Interrupted`. #[deprecated = "use `.save().with_dir(dir)` instead"] pub fn save_in>(&mut self, dir: P) -> io::Result { - self.save().with_dir(dir.as_ref()) + self.save().with_dir(dir.as_ref()).into_result_strict() } /// Save this file to `path`, **truncated** to `limit` (no more than `limit` bytes will be written out). @@ -479,7 +468,7 @@ impl MultipartFile where M: ReadEntry { /// Retries when `io::Error::kind() == io::ErrorKind::Interrupted`. #[deprecated = "use `.save().limit(limit).with_path(path)` instead"] pub fn save_as_limited>(&mut self, path: P, limit: u64) -> io::Result { - self.save().limit(limit).with_path(path) + self.save().limit(limit).with_path(path).into_result_strict() } /// Save this file in the directory pointed at by `dir`, @@ -494,7 +483,7 @@ impl MultipartFile where M: ReadEntry { /// Retries when `io::Error::kind() == io::ErrorKind::Interrupted`. #[deprecated = "use `.save().limit(limit).with_dir(dir)` instead"] pub fn save_in_limited>(&mut self, dir: P, limit: u64) -> io::Result { - self.save().limit(limit).with_dir(dir) + self.save().limit(limit).with_dir(dir).into_result_strict() } } @@ -543,33 +532,7 @@ fn find_header<'a, 'b>(headers: &'a [StrHeader<'b>], name: &str) -> Option<&'a S } /// Common trait for `Multipart` and `&mut Multipart` -pub trait ReadEntry: PrivReadEntry {} - -impl ReadEntry for T where T: PrivReadEntry {} - -/// Public trait but not re-exported. -pub trait PrivReadEntry: Sized { - type Source: BufRead; - - fn source(&mut self) -> &mut Self::Source; - - /// Consume the next boundary. - /// Returns `true` if the last boundary was read, `false` otherwise. - fn consume_boundary(&mut self) -> io::Result; - - fn read_headers(&mut self) -> io::Result> { - FieldHeaders::read_from(&mut self.source()) - } - - fn read_to_string(&mut self) -> io::Result { - let mut buf = String::new(); - - match self.source().read_to_string(&mut buf) { - Ok(_) => Ok(buf), - Err(err) => Err(err), - } - } - +pub trait ReadEntry: PrivReadEntry + Sized { fn read_entry(mut self) -> ReadEntryResult { if try_read_entry!(self; self.consume_boundary()) { return End(self); @@ -621,6 +584,32 @@ pub trait PrivReadEntry: Sized { } } +impl ReadEntry for T where T: PrivReadEntry {} + +/// Public trait but not re-exported. +pub trait PrivReadEntry { + type Source: BufRead; + + fn source(&mut self) -> &mut Self::Source; + + /// Consume the next boundary. + /// Returns `true` if the last boundary was read, `false` otherwise. + fn consume_boundary(&mut self) -> io::Result; + + fn read_headers(&mut self) -> io::Result> { + FieldHeaders::read_from(&mut self.source()) + } + + fn read_to_string(&mut self) -> io::Result { + let mut buf = String::new(); + + match self.source().read_to_string(&mut buf) { + Ok(_) => Ok(buf), + Err(err) => Err(err), + } + } +} + impl<'a, M: ReadEntry> PrivReadEntry for &'a mut M { type Source = M::Source; diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index b9abc3637..a757f4bba 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -118,7 +118,7 @@ impl Multipart { /// If the previously returned entry had contents of type `MultipartField::File`, /// calling this again will discard any unread contents of that entry. pub fn read_entry(&mut self) -> io::Result>> { - PrivReadEntry::read_entry(self).into_result() + ReadEntry::read_entry(self).into_result() } /// Read the next entry from this multipart request, returning a struct with the field's name and @@ -143,7 +143,7 @@ impl Multipart { } } - pub fn save(&mut self) -> SaveBuilder { + pub fn save(&mut self) -> SaveBuilder<&mut Self> { SaveBuilder::new(self) } diff --git a/multipart/src/server/save.rs b/multipart/src/server/save.rs index c50d11d13..2e9944e2e 100644 --- a/multipart/src/server/save.rs +++ b/multipart/src/server/save.rs @@ -10,7 +10,7 @@ use super::buf_redux::copy_buf; use mime::Mime; -use super::field::{MultipartData, MultipartFile}; +use super::field::{MultipartData, MultipartFile, ReadEntry, ReadEntryResult}; use super::Multipart; use self::SaveResult::*; @@ -44,15 +44,6 @@ macro_rules! try_start ( ) ); -macro_rules! try_partial ( - ($try:expr; $partial:expr) => ( - match $try { - Ok(val) => val, - Err(e) => Partial($partial, e.into()), - } - ) -); - /// A builder for saving a file or files to the local filesystem. /// /// ### `OpenOptions` @@ -86,17 +77,17 @@ macro_rules! try_partial ( /// If it is truly necessary, you should sanitize user input such that it cannot cause a path to be /// misinterpreted by the OS. Such functionality is outside the scope of this crate. #[must_use = "nothing saved to the filesystem yet"] -pub struct SaveBuilder<'s, S: 's> { - savable: &'s mut S, +pub struct SaveBuilder { + savable: S, open_opts: OpenOptions, limit: Option, count_limit: Option, } -impl<'s, S: 's> SaveBuilder<'s, S> { +impl SaveBuilder { /// Implementation detail but not problematic to have accessible. #[doc(hidden)] - pub fn new(savable: &'s mut S) -> SaveBuilder<'s, S> { + pub fn new(savable: S) -> SaveBuilder { let mut open_opts = OpenOptions::new(); open_opts.write(true).create_new(true); @@ -127,7 +118,7 @@ impl<'s, S: 's> SaveBuilder<'s, S> { } } -impl<'s, R: 's> SaveBuilder<'s, Multipart> where R: Read { +impl<'m, R: 'm> SaveBuilder<&'m mut Multipart> where R: Read { /// Set the maximum number of files to write out. /// /// Can be `u32` or `Option`. If `None`, clears the limit. @@ -143,7 +134,7 @@ impl<'s, R: 's> SaveBuilder<'s, Multipart> where R: Read { /// /// ### Note: Temporary /// See `SaveDir` for more info (the type of `Entries::save_dir`). - pub fn temp(&mut self) -> EntriesSaveResult<'s, R> { + pub fn temp(&mut self) -> EntriesSaveResult { self.temp_with_prefix("multipart-rs") } @@ -154,7 +145,7 @@ impl<'s, R: 's> SaveBuilder<'s, Multipart> where R: Read { /// /// ### Note: Temporary /// See `SaveDir` for more info (the type of `Entries::save_dir`). - pub fn temp_with_prefix(&mut self, prefix: &str) -> EntriesSaveResult<'s, R> { + pub fn temp_with_prefix(&mut self, prefix: &str) -> EntriesSaveResult { match TempDir::new(prefix) { Ok(tempdir) => self.with_temp_dir(tempdir), Err(e) => SaveResult::Error(e), @@ -164,14 +155,14 @@ impl<'s, R: 's> SaveBuilder<'s, Multipart> where R: Read { /// Save the file fields to the given `TempDir`. /// /// The `TempDir` is returned in the result under `Entries::save_dir`. - pub fn with_temp_dir(&mut self, tempdir: TempDir) -> EntriesSaveResult<'s, R> { + pub fn with_temp_dir(&mut self, tempdir: TempDir) -> EntriesSaveResult { self.with_entries(Entries::new(SaveDir::Temp(tempdir))) } /// Save the file fields in the request to a new permanent directory with the given path. /// /// Any nonexistent parent directories will be created. - pub fn with_dir>(&mut self, dir: P) -> EntriesSaveResult<'s, R> { + pub fn with_dir>(&mut self, dir: P) -> EntriesSaveResult { let dir = dir.into(); try_start!(create_dir_all(&dir)); @@ -179,27 +170,32 @@ impl<'s, R: 's> SaveBuilder<'s, Multipart> where R: Read { self.with_entries(Entries::new(SaveDir::Perm(dir.into()))) } - pub fn with_entries(&mut self, mut entries: Entries) -> EntriesSaveResult<'s, R> { + pub fn with_entries(&mut self, mut entries: Entries) -> EntriesSaveResult { let mut count = 0; loop { - let field = match try_partial!(self.savable.read_entry(); PartialEntries { - entries: entries, - partial_field: None, - }) { - Some(field) => field, - None => break, + let field = match ReadEntry::read_entry(self.savable) { + ReadEntryResult::Entry(field) => field, + ReadEntryResult::End(_) => break, + ReadEntryResult::Error(_, e) => return Partial ( + PartialEntries { + entries: entries, + partial_file: None, + }, + e.into(), + ) }; match field.data { MultipartData::File(mut file) => { match self.count_limit { - Some(limit) if count >= limit => return SaveResult::Partial ( + Some(limit) if count >= limit => return Partial ( PartialEntries { entries: entries, - partial_field: Some(PartialFileField { + partial_file: Some(PartialFileField { field_name: field.name, - file: PartialFile::just_file(file) + source: file, + dest: None, }) }, PartialReason::CountLimit, @@ -210,22 +206,25 @@ impl<'s, R: 's> SaveBuilder<'s, Multipart> where R: Read { count += 1; match file.save().limit(self.limit).with_dir(&entries.save_dir) { - Full(saved_file) => entries.mut_files_for(&field.name).push(saved_file), + Full(saved_file) => entries.mut_files_for(field.name).push(saved_file), Partial(partial, reason) => return Partial( PartialEntries { entries: entries, - partial_field: Some(PartialFileField { + partial_file: Some(PartialFileField { field_name: field.name, - file: partial.with_file(file), + source: file, + dest: Some(partial) }) - } + }, + reason ), - Err(e) => return Partial( + Error(e) => return Partial( PartialEntries { entries: entries, - partial_field: Some(PartialFileField { + partial_file: Some(PartialFileField { field_name: field.name, - file: PartialFile::just_file(file) + source: file, + dest: None, }), }, e.into(), @@ -242,7 +241,7 @@ impl<'s, R: 's> SaveBuilder<'s, Multipart> where R: Read { } } -impl<'s, M: 's> SaveBuilder<'s, MultipartFile> where MultipartFile: BufRead { +impl<'m, M: 'm> SaveBuilder<&'m mut MultipartFile> where MultipartFile: BufRead { /// Save to a file with a random alphanumeric name in the OS temporary directory. /// @@ -304,20 +303,19 @@ impl<'s, M: 's> SaveBuilder<'s, MultipartFile> where MultipartFile: BufRea pub fn with_path>(&mut self, path: P) -> FileSaveResult { let path = path.into(); + let mut saved = SavedFile { + content_type: self.savable.content_type.clone(), + filename: self.savable.filename.clone(), + path: path, + size: 0, + }; + let file = match create_dir_all(&path).and_then(|_| self.open_opts.open(&path)) { Ok(file) => file, - Err(e) => return Partial( - PartialFile { - file_path: Some(path), - file: (), - written: 0, - __priv: (), - }, - e.into(), - ) + Err(e) => return Partial(saved, e.into()) }; - self.write_to(file) + self.write_to(file).map(move |written| saved.with_size(written)) } @@ -384,6 +382,12 @@ pub struct SavedFile { pub size: u64, } +impl SavedFile { + fn with_size(self, size: u64) -> Self { + SavedFile { size: size, .. self } + } +} + /// A result of `Multipart::save_all()`. #[derive(Debug)] pub struct Entries { @@ -404,7 +408,11 @@ impl Entries { } } - pub fn mut_files_for(&mut self, field: &str) -> &mut Vec { + pub fn is_empty(&self) -> bool { + self.fields.is_empty() && self.files.is_empty() + } + + pub fn mut_files_for(&mut self, field: String) -> &mut Vec { self.files.entry(field).or_insert_with(Vec::new) } } @@ -490,45 +498,8 @@ impl AsRef for SaveDir { } } -/// The file that was to be read next when the limit was hit. -#[derive(Clone, Debug)] -pub struct PartialFile { - /// The path of the file on the filesystem. - /// - /// If an error occurred while creating the file, this will not exist. - pub file_path: Option, - - /// The file in the multipart stream. - pub file: F, - - /// The number of bytes written to the filesystem. - pub written: u64, - - __priv: (), -} - -impl PartialFile { - fn with_file(self, file: F_) -> PartialFile { - PartialFile { - file_path: self.file_path, - file: file, - written: self.written, - __priv: () - } - } - - fn just_file(file: F) -> Self { - PartialFile { - file_path: None, - file: file, - written: 0, - __priv: () - } - } -} - /// The reason the save operation quit partway through. -#[derive(Debug, Clone)] +#[derive(Debug)] pub enum PartialReason { /// The count limit for files in the request was hit. /// @@ -551,13 +522,19 @@ impl From for PartialReason { pub struct PartialFileField<'m, R: 'm> { /// The field name for the errored file. pub field_name: String, - pub file: PartialFile>>, - + pub source: MultipartFile<&'m mut Multipart>, + pub dest: Option, } pub struct PartialEntries<'m, R: 'm> { pub entries: Entries, - pub partial_field: Option>, + pub partial_file: Option>, +} + +impl<'m, R: 'm> Into for PartialEntries<'m, R> { + fn into(self) -> Entries { + self.entries + } } /// The result of [`Multipart::save_all()`](struct.multipart.html#method.save_all) @@ -578,36 +555,73 @@ pub type EntriesSaveResult<'m, R> = SaveResult>; /// Shorthand result for methods that return `SavedFile`s. /// -/// The `MultipartFile` is not available here because it is not necessary to return -/// a borrow when the owned version is probably in the same scope. -pub type FileSaveResult = SaveResult>; +/// The `MultipartFile` is not provided here because it is not necessary to return +/// a borrow when the owned version is probably in the same scope. This hopefully +/// saves some headache with the borrow-checker. +pub type FileSaveResult = SaveResult; impl<'m, R> EntriesSaveResult<'m, R> { /// Take the `Entries` from `self`, if applicable, and discarding /// the error, if any. - pub fn to_entries(self) -> Option { + pub fn into_entries(self) -> Option { match self { Full(entries) | Partial(PartialEntries { entries, .. }, _) => Some(entries), Error(_) => None, } } +} - /// Decompose `self` to `(Option, Option)` - pub fn to_opt(self) -> (Option, Option) { +impl SaveResult where P: Into { + /// Convert `self` to `Option`; there may still have been an error. + pub fn okish(self) -> Option { + self.into_opt_both().0 + } + + /// Map the `Full` or `Partial` values to a new type, retaining the reason + /// in the `Partial` case. + pub fn map(self, map: Map) -> SaveResult where Map: FnOnce(S) -> T { + match self { + Full(full) => Full(map(full)), + Partial(partial, reason) => Partial(map(partial.into()), reason), + Error(e) => Error(e), + } + } + + /// Decompose `self` to `(Option, Option)` + pub fn into_opt_both(self) -> (Option, Option) { match self { - Partial(PartialEntries { entries, .. }, PartialReason::IoError(e)) => (Some(entries), Some(e)), - Full(entries) | Partial(entries, _) => (Some(entries), None), + Full(full) => (Some(full), None), + Partial(partial, _) => (Some(partial.into()), None), + Partial(partial, PartialReason::IoError(e)) => (Some(partial.into()), Some(e)), Error(error) => (None, Some(error)), } } /// Map `self` to an `io::Result`, discarding the error in the `Partial` case. - pub fn to_result(self) -> io::Result { + pub fn into_result(self) -> io::Result { match self { - Full(entries) | Partial(PartialEntries { entries, .. }, _) => Ok(entries), + Full(entries) => Ok(entries), + Partial(partial, _) => Ok(partial.into()), Error(error) => Err(error), } } + + /// Pessimistic version of `into_result()` which will return an error even + /// for the `Partial` case. + /// + /// ### Note: Possible Storage Leak + /// It's generally not a good idea to ignore the `Partial` case, as there may still be a + /// partially written file on-disk. If you're not using a temporary directory + /// (OS-managed or via `TempDir`) then partially written files will remain on-disk until + /// explicitly removed which could result in excessive disk usage if not monitored closely. + pub fn into_result_strict(self) -> io::Result { + match self { + Full(entries) => Ok(entries), + Partial(_, PartialReason::IoError(e)) => Err(e), + Partial(partial, _) => Ok(partial.into()), + Error(e) => Err(e), + } + } } fn create_dir_all(path: &Path) -> io::Result<()> { @@ -621,7 +635,7 @@ fn create_dir_all(path: &Path) -> io::Result<()> { } fn try_copy_buf(mut src: R, dest: W) -> SaveResult { - let mut total_copied = 0; + let mut total_copied = 0u64; macro_rules! try_here ( ($try:expr) => ( @@ -637,6 +651,8 @@ fn try_copy_buf(mut src: R, dest: W) -> SaveResult try_here!(Err(io::Error::new(io::ErrorKind::WriteZero, From 2d9eb6203377bd91bfb742fef62388d423be4b29 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sat, 25 Feb 2017 10:59:41 -0800 Subject: [PATCH 245/453] WIP everything compiles now --- multipart/src/server/field.rs | 16 +++-- multipart/src/server/iron.rs | 118 ++++++++++++++++++--------------- multipart/src/server/mod.rs | 12 ++-- multipart/src/server/save.rs | 121 ++++++++++++++++++++++++---------- 4 files changed, 164 insertions(+), 103 deletions(-) diff --git a/multipart/src/server/field.rs b/multipart/src/server/field.rs index 86083fb71..723c4e399 100644 --- a/multipart/src/server/field.rs +++ b/multipart/src/server/field.rs @@ -305,8 +305,9 @@ impl Into for MultipartText { } impl MultipartText { - fn take_inner(&mut self) -> M { - self.inner.take().expect("MultipartText::inner taken!") + #[doc(hidden)] + pub fn take_inner(&mut self) -> M { + self.inner.take().expect("MultipartText::inner already taken!") } fn into_inner(self) -> M { @@ -400,8 +401,9 @@ impl MultipartFile { self.inner.as_mut().expect("MultipartFile::inner taken!") } - fn take_inner(&mut self) -> M { - self.inner.take().expect("MultipartFile::inner taken!") + #[doc(hidden)] + pub fn take_inner(&mut self) -> M { + self.inner.take().expect("MultipartFile::inner already taken!") } fn into_inner(self) -> M { @@ -433,7 +435,7 @@ impl MultipartFile where M: ReadEntry { /// Retries when `io::Error::kind() == io::ErrorKind::Interrupted`. #[deprecated = "use `.save().limit(limit).write_to(out)` instead"] pub fn save_to_limited(&mut self, out: W, limit: u64) -> io::Result { - self.save().limit(limit).write_to(out).into_result_strict() + self.save().size_limit(limit).write_to(out).into_result_strict() } /// Save this file to `path`. @@ -468,7 +470,7 @@ impl MultipartFile where M: ReadEntry { /// Retries when `io::Error::kind() == io::ErrorKind::Interrupted`. #[deprecated = "use `.save().limit(limit).with_path(path)` instead"] pub fn save_as_limited>(&mut self, path: P, limit: u64) -> io::Result { - self.save().limit(limit).with_path(path).into_result_strict() + self.save().size_limit(limit).with_path(path).into_result_strict() } /// Save this file in the directory pointed at by `dir`, @@ -483,7 +485,7 @@ impl MultipartFile where M: ReadEntry { /// Retries when `io::Error::kind() == io::ErrorKind::Interrupted`. #[deprecated = "use `.save().limit(limit).with_dir(dir)` instead"] pub fn save_in_limited>(&mut self, dir: P, limit: u64) -> io::Result { - self.save().limit(limit).with_dir(dir).into_result_strict() + self.save().size_limit(limit).with_dir(dir).into_result_strict() } } diff --git a/multipart/src/server/iron.rs b/multipart/src/server/iron.rs index 9ec080a96..6e6bfad3a 100644 --- a/multipart/src/server/iron.rs +++ b/multipart/src/server/iron.rs @@ -10,10 +10,11 @@ use iron::typemap::Key; use iron::{BeforeMiddleware, IronError, IronResult}; use std::path::PathBuf; -use std::{error, fmt}; +use std::{error, fmt, io}; use super::{HttpRequest, Multipart, MultipartData}; -use super::save::Entries; +use super::save::{Entries, PartialReason, SaveBuilder, TempDir}; +use super::save::SaveResult::*; impl<'r, 'a, 'b> HttpRequest for &'r mut IronRequest<'a, 'b> { type Body = &'r mut IronBody<'a, 'b>; @@ -118,69 +119,74 @@ impl Intercept { } fn read_request(&self, req: &mut IronRequest) -> IronResult> { - macro_rules! try_iron( - ($try:expr; $($fmt_args:tt)+) => ( - match $try { - Ok(ok) => ok, - Err(err) => return Err(IronError::new(err, format!($($fmt_args)*))), - } - ); - ($try:expr) => ( - try_iron!($try; ""); - ) - ); - let mut multipart = match Multipart::from_request(req) { Ok(multipart) => multipart, Err(_) => return Ok(None), }; - let mut entries = try_iron!( + let tempdir = try!( self.temp_dir_path.as_ref() - .map_or_else(Entries::new_tempdir, Entries::new_tempdir_in); - "Error opening temporary directory for request." - ); - - let mut file_count = 0; - - while let Some(field) = try_iron!(multipart.read_entry()) { - match field.data { - MultipartData::File(mut file) => { - if self.limit_behavior.throw_error() && file_count >= self.file_count_limit { - return Err(FileCountLimitError(self.file_count_limit).into()); - } - - let file = try_iron!( - file.save().limit(self.file_size_limit).with_dir(&entries.dir); - "Error reading field: \"{}\" (filename: \"{}\")", - field.name, - file.filename.as_ref().map_or("(none)", |f| f) - ); - - if file.size == self.file_size_limit { - if self.limit_behavior.throw_error() { - return Err(FileSizeLimitError::new(field.name, file.filename).into()); - } else { - warn!( - "File size limit reached for field {:?} (filename: {:?})", - field.name, file.filename - ); - } - } - - entries.files.insert(field.name, file); - file_count += 1; - }, - MultipartData::Text(text) => { - entries.fields.insert(field.name, text.text); - }, - } + .map_or_else( + || TempDir::new("multipart-iron"), + |path| TempDir::new_in(path, "multipart-iron") + ) + .map_err(|e| io_to_iron(e, "Error opening temporary directory for request.")) + ); + + match self.limit_behavior { + LimitBehavior::ThrowError => self.read_request_strict(multipart, tempdir), + LimitBehavior::Continue => self.read_request_lenient(multipart, tempdir), } + } - Ok(Some(entries)) + fn read_request_strict(&self, mut multipart: IronMultipart, tempdir: TempDir) -> IronResult> { + match multipart.save().size_limit(self.file_size_limit) + .count_limit(self.file_count_limit) + .with_temp_dir(tempdir) { + Full(entries) => Ok(Some(entries)), + Partial(_, PartialReason::IoError(err)) => Err(io_to_iron(err, "Error midway through request")), + Partial(_, PartialReason::CountLimit) => Err(FileCountLimitError(self.file_count_limit).into()), + Partial(partial, PartialReason::SizeLimit) => { + let partial_file = partial.partial_file.expect(EXPECT_PARTIAL_FILE); + Err( + FileSizeLimitError { + field: partial_file.field_name, + filename: partial_file.source.filename, + }.into() + ) + }, + Error(err) => Err(io_to_iron(err, "Error at start of request")), + } + } + + fn read_request_lenient(&self, mut multipart: IronMultipart, tempdir: TempDir) -> IronResult> { + let mut entries = match multipart.save().size_limit(self.file_size_limit) + .count_limit(self.file_count_limit) + .with_temp_dir(tempdir) { + Full(entries) => return Ok(Some(entries)), + Partial(_, PartialReason::IoError(err)) => return Err(io_to_iron(err, "Error midway through request")), + Partial(partial, _) => partial.keep_partial(), + Error(err) => return Err(io_to_iron(err, "Error at start of request")), + }; + + loop { + entries = match multipart.save().size_limit(self.file_size_limit) + .count_limit(self.file_count_limit) + .with_entries(entries) { + Full(entries) => return Ok(Some(entries)), + Partial(_, PartialReason::IoError(err)) => return Err(io_to_iron(err, "Error midway through request")), + Partial(partial, _) => partial.keep_partial(), + Error(err) => return Err(io_to_iron(err, "Error at start of request")), + }; + } } } +type IronMultipart<'r, 'a, 'b> = Multipart<&'r mut IronBody<'a, 'b>>; + +const EXPECT_PARTIAL_FILE: &'static str = "File size limit hit but the offending \ + file was not available; this is a bug."; + impl Default for Intercept { fn default() -> Self { Intercept { @@ -295,3 +301,7 @@ impl Into for FileCountLimitError { IronError::new(self, desc_string) } } + +fn io_to_iron>(err: io::Error, msg: M) -> IronError { + IronError::new(err, msg.into()) +} diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index a757f4bba..09980b6ee 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -153,7 +153,7 @@ impl Multipart { /// If there is an error in reading the request, returns the partial result along with the /// error. See [`SaveResult`](enum.SaveResult.html) for more information. #[deprecated = "use `.save().temp()` instead"] - pub fn save_all(&mut self) -> EntriesSaveResult { + pub fn save_all(&mut self) -> EntriesSaveResult<&mut Self> { self.save().temp() } @@ -163,7 +163,7 @@ impl Multipart { /// If there is an error in reading the request, returns the partial result along with the /// error. See [`SaveResult`](enum.SaveResult.html) for more information. #[deprecated = "use `.save().with_temp_dir()` instead"] - pub fn save_all_under>(&mut self, dir: P) -> EntriesSaveResult { + pub fn save_all_under>(&mut self, dir: P) -> EntriesSaveResult<&mut Self> { match TempDir::new_in(dir, "multipart") { Ok(temp_dir) => self.save().with_temp_dir(temp_dir), Err(err) => return SaveResult::Error(err), @@ -178,8 +178,8 @@ impl Multipart { /// If there is an error in reading the request, returns the partial result along with the /// error. See [`SaveResult`](enum.SaveResult.html) for more information. #[deprecated = "use `.save().limit(limit)` instead"] - pub fn save_all_limited(&mut self, limit: u64) -> EntriesSaveResult { - self.save().limit(limit).temp() + pub fn save_all_limited(&mut self, limit: u64) -> EntriesSaveResult<&mut Self> { + self.save().size_limit(limit).temp() } /// Read the request fully, parsing all fields and saving all files in a new temporary @@ -190,9 +190,9 @@ impl Multipart { /// If there is an error in reading the request, returns the partial result along with the /// error. See [`SaveResult`](enum.SaveResult.html) for more information. #[deprecated = "use `.save().limit(limit).with_temp_dir()` instead"] - pub fn save_all_under_limited>(&mut self, dir: P, limit: u64) -> EntriesSaveResult { + pub fn save_all_under_limited>(&mut self, dir: P, limit: u64) -> EntriesSaveResult<&mut Self> { match TempDir::new_in(dir, "multipart") { - Ok(temp_dir) => self.save().limit(limit).with_temp_dir(temp_dir), + Ok(temp_dir) => self.save().size_limit(limit).with_temp_dir(temp_dir), Err(err) => return SaveResult::Error(err), } } diff --git a/multipart/src/server/save.rs b/multipart/src/server/save.rs index 2e9944e2e..ade811ec9 100644 --- a/multipart/src/server/save.rs +++ b/multipart/src/server/save.rs @@ -80,7 +80,7 @@ macro_rules! try_start ( pub struct SaveBuilder { savable: S, open_opts: OpenOptions, - limit: Option, + size_limit: Option, count_limit: Option, } @@ -94,7 +94,7 @@ impl SaveBuilder { SaveBuilder { savable: savable, open_opts: open_opts, - limit: None, + size_limit: None, count_limit: None, } } @@ -102,8 +102,8 @@ impl SaveBuilder { /// Set the maximum number of bytes to write out *per file*. /// /// Can be `u64` or `Option`. If `None`, clears the limit. - pub fn limit>>(&mut self, limit: L) -> &mut Self { - self.limit = limit.into(); + pub fn size_limit>>(mut self, limit: L) -> Self { + self.size_limit = limit.into(); self } @@ -111,18 +111,18 @@ impl SaveBuilder { /// /// The `write` flag will be reset to `true` after the closure returns. (It'd be pretty /// pointless otherwise, right?) - pub fn mod_open_opts(&mut self, opts_fn: F) -> &mut Self { + pub fn mod_open_opts(mut self, opts_fn: F) -> Self { opts_fn(&mut self.open_opts); self.open_opts.write(true); self } } -impl<'m, R: 'm> SaveBuilder<&'m mut Multipart> where R: Read { +impl SaveBuilder where M: ReadEntry { /// Set the maximum number of files to write out. /// /// Can be `u32` or `Option`. If `None`, clears the limit. - pub fn count_limit>>(&mut self, count_limit: L) -> &mut Self { + pub fn count_limit>>(mut self, count_limit: L) -> Self { self.count_limit = count_limit.into(); self } @@ -134,7 +134,7 @@ impl<'m, R: 'm> SaveBuilder<&'m mut Multipart> where R: Read { /// /// ### Note: Temporary /// See `SaveDir` for more info (the type of `Entries::save_dir`). - pub fn temp(&mut self) -> EntriesSaveResult { + pub fn temp(self) -> EntriesSaveResult { self.temp_with_prefix("multipart-rs") } @@ -145,7 +145,7 @@ impl<'m, R: 'm> SaveBuilder<&'m mut Multipart> where R: Read { /// /// ### Note: Temporary /// See `SaveDir` for more info (the type of `Entries::save_dir`). - pub fn temp_with_prefix(&mut self, prefix: &str) -> EntriesSaveResult { + pub fn temp_with_prefix(self, prefix: &str) -> EntriesSaveResult { match TempDir::new(prefix) { Ok(tempdir) => self.with_temp_dir(tempdir), Err(e) => SaveResult::Error(e), @@ -155,14 +155,14 @@ impl<'m, R: 'm> SaveBuilder<&'m mut Multipart> where R: Read { /// Save the file fields to the given `TempDir`. /// /// The `TempDir` is returned in the result under `Entries::save_dir`. - pub fn with_temp_dir(&mut self, tempdir: TempDir) -> EntriesSaveResult { + pub fn with_temp_dir(self, tempdir: TempDir) -> EntriesSaveResult { self.with_entries(Entries::new(SaveDir::Temp(tempdir))) } /// Save the file fields in the request to a new permanent directory with the given path. /// /// Any nonexistent parent directories will be created. - pub fn with_dir>(&mut self, dir: P) -> EntriesSaveResult { + pub fn with_dir>(self, dir: P) -> EntriesSaveResult { let dir = dir.into(); try_start!(create_dir_all(&dir)); @@ -170,7 +170,7 @@ impl<'m, R: 'm> SaveBuilder<&'m mut Multipart> where R: Read { self.with_entries(Entries::new(SaveDir::Perm(dir.into()))) } - pub fn with_entries(&mut self, mut entries: Entries) -> EntriesSaveResult { + pub fn with_entries(mut self, mut entries: Entries) -> EntriesSaveResult { let mut count = 0; loop { @@ -205,8 +205,11 @@ impl<'m, R: 'm> SaveBuilder<&'m mut Multipart> where R: Read { count += 1; - match file.save().limit(self.limit).with_dir(&entries.save_dir) { - Full(saved_file) => entries.mut_files_for(field.name).push(saved_file), + match file.save().size_limit(self.size_limit).with_dir(&entries.save_dir) { + Full(saved_file) => { + self.savable = file.take_inner(); + entries.mut_files_for(field.name).push(saved_file); + }, Partial(partial, reason) => return Partial( PartialEntries { entries: entries, @@ -231,7 +234,8 @@ impl<'m, R: 'm> SaveBuilder<&'m mut Multipart> where R: Read { ), } }, - MultipartData::Text(text) => { + MultipartData::Text(mut text) => { + self.savable = text.take_inner(); entries.fields.insert(field.name, text.text); }, } @@ -310,7 +314,7 @@ impl<'m, M: 'm> SaveBuilder<&'m mut MultipartFile> where MultipartFile: Bu size: 0, }; - let file = match create_dir_all(&path).and_then(|_| self.open_opts.open(&path)) { + let file = match create_dir_all(&saved.path).and_then(|_| self.open_opts.open(&saved.path)) { Ok(file) => file, Err(e) => return Partial(saved, e.into()) }; @@ -326,7 +330,7 @@ impl<'m, M: 'm> SaveBuilder<&'m mut MultipartFile> where MultipartFile: Bu /// /// Retries on interrupts. pub fn write_to(&mut self, mut dest: W) -> SaveResult { - if let Some(limit) = self.limit { + if let Some(limit) = self.size_limit { let copied = match try_copy_buf(self.savable.take(limit), &mut dest) { Full(copied) => copied, other => return other, @@ -519,24 +523,40 @@ impl From for PartialReason { } } -pub struct PartialFileField<'m, R: 'm> { +pub struct PartialFileField { /// The field name for the errored file. pub field_name: String, - pub source: MultipartFile<&'m mut Multipart>, + pub source: MultipartFile, pub dest: Option, } -pub struct PartialEntries<'m, R: 'm> { +pub struct PartialEntries { pub entries: Entries, - pub partial_file: Option>, + pub partial_file: Option>, } -impl<'m, R: 'm> Into for PartialEntries<'m, R> { +impl Into for PartialEntries { fn into(self) -> Entries { self.entries } } +impl PartialEntries { + /// If `partial_file` is present and contains a `SavedFile` then just + /// add it to the `Entries` instance and return it. + /// + /// Otherwise, returns `self.entries` + pub fn keep_partial(mut self) -> Entries { + if let Some(partial_file) = self.partial_file { + if let Some(saved_file) = partial_file.dest { + self.entries.mut_files_for(partial_file.field_name).push(saved_file); + } + } + + self.entries + } +} + /// The result of [`Multipart::save_all()`](struct.multipart.html#method.save_all) /// and methods on `SaveBuilder`. #[derive(Debug)] @@ -551,7 +571,7 @@ pub enum SaveResult { } /// Shorthand result for methods that return `Entries` -pub type EntriesSaveResult<'m, R> = SaveResult>; +pub type EntriesSaveResult = SaveResult>; /// Shorthand result for methods that return `SavedFile`s. /// @@ -560,7 +580,7 @@ pub type EntriesSaveResult<'m, R> = SaveResult>; /// saves some headache with the borrow-checker. pub type FileSaveResult = SaveResult; -impl<'m, R> EntriesSaveResult<'m, R> { +impl EntriesSaveResult { /// Take the `Entries` from `self`, if applicable, and discarding /// the error, if any. pub fn into_entries(self) -> Option { @@ -634,7 +654,7 @@ fn create_dir_all(path: &Path) -> io::Result<()> { } } -fn try_copy_buf(mut src: R, dest: W) -> SaveResult { +fn try_copy_buf(mut src: R, mut dest: W) -> SaveResult { let mut total_copied = 0u64; macro_rules! try_here ( @@ -649,20 +669,49 @@ fn try_copy_buf(mut src: R, dest: W) -> SaveResult { src.consume(copied); total_copied += copied as u64; } + Partial(copied, reason) => { + src.consume(copied); total_copied += copied as u64; + return Partial(total_copied, reason); + }, + Error(err) => { + return Partial(total_copied, err.into()); + } + } + } - while !buf.is_empty() { - match try_here!(dest.write(buf)) { - 0 => try_here!(Err(io::Error::new(io::ErrorKind::WriteZero, - "failed to write whole buffer"))), - copied => { - buf = &mut buf[copied..]; - total_copied += copied as u64; - src.consume(copied) - }, + Full(total_copied) +} + +fn try_write_all(mut buf: &[u8], mut dest: W) -> SaveResult where W: Write { + let mut total_copied = 0; + + macro_rules! try_here ( + ($try:expr) => ( + match $try { + Ok(val) => val, + Err(ref e) if e.kind() == io::ErrorKind::Interrupted => continue, + Err(e) => return if total_copied == 0 { Error(e) } + else { Partial(total_copied, e.into()) }, } + ) + ); + + while !buf.is_empty() { + match try_here!(dest.write(buf)) { + 0 => try_here!(Err(io::Error::new(io::ErrorKind::WriteZero, + "failed to write whole buffer"))), + copied => { + buf = &buf[copied..]; + total_copied += copied; + }, } } From 7b698cd2a57f4f9ecb108a828fead8d4dce562aa Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sat, 25 Feb 2017 11:09:34 -0800 Subject: [PATCH 246/453] Fixup code --- multipart/src/server/field.rs | 5 ++--- multipart/src/server/iron.rs | 26 +++----------------------- multipart/src/server/mod.rs | 6 ++---- multipart/src/server/save.rs | 15 +++------------ 4 files changed, 10 insertions(+), 42 deletions(-) diff --git a/multipart/src/server/field.rs b/multipart/src/server/field.rs index 723c4e399..7efcdf3ea 100644 --- a/multipart/src/server/field.rs +++ b/multipart/src/server/field.rs @@ -11,15 +11,14 @@ use super::httparse::{self, EMPTY_HEADER, Status}; use self::ReadEntryResult::*; -use super::save::{PartialReason, SaveBuilder, SavedFile}; +use super::save::{SaveBuilder, SavedFile}; use mime::{TopLevel, Mime}; -use std::fs::{self, OpenOptions}; use std::io::{self, Read, BufRead, Write}; use std::ops::Deref; use std::path::{Path, PathBuf}; -use std::{env, str}; +use std::str; macro_rules! try_io( ($try:expr) => ( diff --git a/multipart/src/server/iron.rs b/multipart/src/server/iron.rs index 6e6bfad3a..c1195e3c5 100644 --- a/multipart/src/server/iron.rs +++ b/multipart/src/server/iron.rs @@ -12,8 +12,8 @@ use iron::{BeforeMiddleware, IronError, IronResult}; use std::path::PathBuf; use std::{error, fmt, io}; -use super::{HttpRequest, Multipart, MultipartData}; -use super::save::{Entries, PartialReason, SaveBuilder, TempDir}; +use super::{HttpRequest, Multipart}; +use super::save::{Entries, PartialReason, TempDir}; use super::save::SaveResult::*; impl<'r, 'a, 'b> HttpRequest for &'r mut IronRequest<'a, 'b> { @@ -119,7 +119,7 @@ impl Intercept { } fn read_request(&self, req: &mut IronRequest) -> IronResult> { - let mut multipart = match Multipart::from_request(req) { + let multipart = match Multipart::from_request(req) { Ok(multipart) => multipart, Err(_) => return Ok(None), }; @@ -226,17 +226,6 @@ pub enum LimitBehavior { Continue, } -impl LimitBehavior { - fn throw_error(self) -> bool { - use self::LimitBehavior::*; - - match self { - ThrowError => true, - Continue => false, - } - } -} - /// An error returned from `Intercept` when the size limit /// for an individual file is exceeded. #[derive(Debug)] @@ -247,15 +236,6 @@ pub struct FileSizeLimitError { pub filename: Option, } -impl FileSizeLimitError { - fn new(field: String, filename: Option) -> Self { - FileSizeLimitError { - field: field, - filename: filename - } - } -} - impl error::Error for FileSizeLimitError { fn description(&self) -> &str { "file size limit reached" diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index 09980b6ee..194dc4ac5 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -17,11 +17,9 @@ extern crate safemem; extern crate twoway; use std::borrow::Borrow; -use std::collections::HashMap; -use std::fs; use std::io::prelude::*; -use std::path::{Path, PathBuf}; -use std::{io, mem}; +use std::path::Path; +use std::io; use tempdir::TempDir; diff --git a/multipart/src/server/save.rs b/multipart/src/server/save.rs index ade811ec9..7fad327bc 100644 --- a/multipart/src/server/save.rs +++ b/multipart/src/server/save.rs @@ -6,12 +6,9 @@ // copied, modified, or distributed except according to those terms. //! Utilities for saving request entries to the filesystem. -use super::buf_redux::copy_buf; - use mime::Mime; use super::field::{MultipartData, MultipartFile, ReadEntry, ReadEntryResult}; -use super::Multipart; use self::SaveResult::*; @@ -25,12 +22,6 @@ use std::{env, fs, io, mem}; const RANDOM_FILENAME_LEN: usize = 12; -// Because this isn't exposed as a str in the stdlib -#[cfg(not(windows))] -const PATH_SEP: &'static str = "/"; -#[cfg(windows)] -const PATH_SEP: &'static str = "\\"; - fn rand_filename() -> String { ::random_alphanumeric(RANDOM_FILENAME_LEN) } @@ -307,7 +298,7 @@ impl<'m, M: 'm> SaveBuilder<&'m mut MultipartFile> where MultipartFile: Bu pub fn with_path>(&mut self, path: P) -> FileSaveResult { let path = path.into(); - let mut saved = SavedFile { + let saved = SavedFile { content_type: self.savable.content_type.clone(), filename: self.savable.filename.clone(), path: path, @@ -611,8 +602,8 @@ impl SaveResult where P: Into { pub fn into_opt_both(self) -> (Option, Option) { match self { Full(full) => (Some(full), None), - Partial(partial, _) => (Some(partial.into()), None), Partial(partial, PartialReason::IoError(e)) => (Some(partial.into()), Some(e)), + Partial(partial, _) => (Some(partial.into()), None), Error(error) => (None, Some(error)), } } @@ -670,7 +661,7 @@ fn try_copy_buf(mut src: R, mut dest: W) -> SaveResult Date: Sun, 26 Feb 2017 12:20:35 -0800 Subject: [PATCH 247/453] Add missing documentation, `Debug` impls --- multipart/src/server/field.rs | 4 +++- multipart/src/server/mod.rs | 1 + multipart/src/server/save.rs | 31 +++++++++++++++++++++++++++---- 3 files changed, 31 insertions(+), 5 deletions(-) diff --git a/multipart/src/server/field.rs b/multipart/src/server/field.rs index 7efcdf3ea..3105d99c5 100644 --- a/multipart/src/server/field.rs +++ b/multipart/src/server/field.rs @@ -534,6 +534,7 @@ fn find_header<'a, 'b>(headers: &'a [StrHeader<'b>], name: &str) -> Option<&'a S /// Common trait for `Multipart` and `&mut Multipart` pub trait ReadEntry: PrivReadEntry + Sized { + /// Attempt to read the next entry in the multipart stream. fn read_entry(mut self) -> ReadEntryResult { if try_read_entry!(self; self.consume_boundary()) { return End(self); @@ -623,7 +624,8 @@ impl<'a, M: ReadEntry> PrivReadEntry for &'a mut M { } } -/// Result type returned by `Multipart::into_entry()` and `MultipartField::next_entry()`. +/// Ternary result type returned by `ReadEntry::next_entry()`, +/// `Multipart::into_entry()` and `MultipartField::next_entry()`. pub enum ReadEntryResult> { /// The next entry was found. Entry(Entry), diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index 194dc4ac5..4bece0005 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -141,6 +141,7 @@ impl Multipart { } } + /// Get a builder type for saving the files in this request to the filesystem. pub fn save(&mut self) -> SaveBuilder<&mut Self> { SaveBuilder::new(self) } diff --git a/multipart/src/server/save.rs b/multipart/src/server/save.rs index 7fad327bc..227c59f90 100644 --- a/multipart/src/server/save.rs +++ b/multipart/src/server/save.rs @@ -161,6 +161,9 @@ impl SaveBuilder where M: ReadEntry { self.with_entries(Entries::new(SaveDir::Perm(dir.into()))) } + /// Commence the save operation using the existing `Entries` instance. + /// + /// May be used to resume a saving operation after handling an error. pub fn with_entries(mut self, mut entries: Entries) -> EntriesSaveResult { let mut count = 0; @@ -403,11 +406,12 @@ impl Entries { } } + /// Returns `true` if both `fields` and `files` are empty, `false` otherwise. pub fn is_empty(&self) -> bool { self.fields.is_empty() && self.files.is_empty() } - pub fn mut_files_for(&mut self, field: String) -> &mut Vec { + fn mut_files_for(&mut self, field: String) -> &mut Vec { self.files.entry(field).or_insert_with(Vec::new) } } @@ -419,6 +423,9 @@ pub enum SaveDir { /// is dropped. Temp(TempDir), /// This directory is permanent and will be left on the filesystem when this wrapper is dropped. + /// + /// **N.B.** If this directory is in the OS temporary directory then it may still be + /// deleted at any time, usually on reboot or when free space is low. Perm(PathBuf), } @@ -514,18 +521,35 @@ impl From for PartialReason { } } +/// The file field that was being read when the save operation quit. +/// +/// May be partially saved to the filesystem if `dest` is `Some`. +#[derive(Debug)] pub struct PartialFileField { - /// The field name for the errored file. + /// The field name for the partial file. pub field_name: String, + /// The partial file's source in the multipart stream (may be partially read if `dest` + /// is `Some`). pub source: MultipartFile, + /// The partial file's entry on the filesystem, if the operation got that far. pub dest: Option, } +/// The partial result type for `Multipart::save*()`. +/// +/// Contains the successfully saved entries as well as the partially +/// saved file that was in the process of being read when the error occurred, +/// if applicable. +#[derive(Debug)] pub struct PartialEntries { + /// The entries that were saved successfully. pub entries: Entries, + /// The file that was in the process of being read. `None` if the error + /// occurred between file entries. pub partial_file: Option>, } +/// Discards `partial_file` impl Into for PartialEntries { fn into(self) -> Entries { self.entries @@ -548,8 +572,7 @@ impl PartialEntries { } } -/// The result of [`Multipart::save_all()`](struct.multipart.html#method.save_all) -/// and methods on `SaveBuilder`. +/// The ternary result type used for the `SaveBuilder<_>` API. #[derive(Debug)] pub enum SaveResult { /// The operation was a total success. Contained is the complete result. From 90611bb077d4cdc675aa546b2f3022c46faf8ebb Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sun, 26 Feb 2017 13:51:41 -0800 Subject: [PATCH 248/453] Get examples and tests compiling again --- multipart/examples/hyper_server.rs | 42 +++++++++++++-------- multipart/examples/iron.rs | 60 +++++++++++++++++------------- multipart/examples/nickel.rs | 47 ++++++++++++----------- multipart/examples/tiny_http.rs | 51 ++++++++++++++----------- multipart/src/local_test.rs | 2 +- multipart/src/server/field.rs | 5 +++ multipart/src/server/mod.rs | 4 +- multipart/src/server/save.rs | 16 ++++++++ 8 files changed, 137 insertions(+), 90 deletions(-) diff --git a/multipart/examples/hyper_server.rs b/multipart/examples/hyper_server.rs index 36e335157..c100a92e5 100644 --- a/multipart/examples/hyper_server.rs +++ b/multipart/examples/hyper_server.rs @@ -2,12 +2,12 @@ extern crate hyper; extern crate multipart; use std::fs::File; -use std::io::{Read, Result}; +use std::io::{self, Read}; use hyper::server::{Handler, Server, Request, Response}; use hyper::status::StatusCode; use hyper::server::response::Response as HyperResponse; use multipart::server::hyper::{Switch, MultipartHandler, HyperRequest}; -use multipart::server::{Multipart, Entries, SaveResult}; +use multipart::server::{Multipart, Entries, SaveResult, SavedFile}; struct NonMultipart; impl Handler for NonMultipart { @@ -20,11 +20,11 @@ impl Handler for NonMultipart { struct EchoMultipart; impl MultipartHandler for EchoMultipart { fn handle_multipart(&self, mut multipart: Multipart, mut res: HyperResponse) { - let processing = match multipart.save_all() { + let processing = match multipart.save().temp() { SaveResult::Full(entries) => process_entries(entries), SaveResult::Partial(entries, error) => { println!("Errors saving multipart:\n{:?}", error); - process_entries(entries) + process_entries(entries.into()) } SaveResult::Error(error) => { println!("Errors saving multipart:\n{:?}", error); @@ -41,24 +41,34 @@ impl MultipartHandler for EchoMultipart { } } -fn process_entries(entries: Entries) -> Result<()> { +fn process_entries<'a>(entries: Entries) -> io::Result<()> { for (name, field) in entries.fields { - print!(r#"Field "{}": "{}""#, name, field); + println!("Field {:?}: {:?}", name, field); } - for (name, savedfile) in entries.files { - let filename = match savedfile.filename { - Some(s) => s, - None => "None".into() - }; - let mut file = try!(File::open(savedfile.path)); - let mut contents = String::new(); - try!(file.read_to_string(&mut contents)); - println!(r#"Field "{}" is file "{}":"#, name, filename); - println!("{}", contents); + for (name, files) in entries.files { + println!("Field {:?} has {} files:", name, files.len()); + + for file in files { + try!(print_file(&file)); + } } + Ok(()) } + +fn print_file(saved_file: &SavedFile) -> io::Result<()> { + let mut file = try!(File::open(&saved_file.path)); + + let mut contents = String::new(); + try!(file.read_to_string(&mut contents)); + + println!("File {:?} ({:?}):", saved_file.filename, saved_file.content_type); + println!("{}", contents); + + Ok(()) +} + fn main() { println!("Listening on 0.0.0.0:3333"); Server::http("0.0.0.0:3333").unwrap().handle( diff --git a/multipart/examples/iron.rs b/multipart/examples/iron.rs index 362b399cb..5a972d464 100644 --- a/multipart/examples/iron.rs +++ b/multipart/examples/iron.rs @@ -3,7 +3,7 @@ extern crate iron; use std::fs::File; use std::io::Read; -use multipart::server::{Multipart, Entries, SaveResult}; +use multipart::server::{Multipart, Entries, SaveResult, SavedFile}; use iron::prelude::*; use iron::status; @@ -17,13 +17,13 @@ fn process_request(request: &mut Request) -> IronResult { match Multipart::from_request(request) { Ok(mut multipart) => { // Fetching all data and processing it. - // save_all() reads the request fully, parsing all fields and saving all files + // save().temp() reads the request fully, parsing all fields and saving all files // in a new temporary directory under the OS temporary directory. - match multipart.save_all() { + match multipart.save().temp() { SaveResult::Full(entries) => process_entries(entries), - SaveResult::Partial(entries, error) => { - try!(process_entries(entries)); - Err(IronError::new(error, status::InternalServerError)) + SaveResult::Partial(entries, reason) => { + try!(process_entries(entries.keep_partial())); + Err(IronError::new(reason.unwrap_err(), status::InternalServerError)) } SaveResult::Error(error) => Err(IronError::new(error, status::InternalServerError)), } @@ -38,29 +38,37 @@ fn process_request(request: &mut Request) -> IronResult { /// Returns an OK response or an error. fn process_entries(entries: Entries) -> IronResult { for (name, field) in entries.fields { - println!(r#"Field "{}": "{}""#, name, field); + println!("Field {:?}: {:?}", name, field); } - for (name, savedfile) in entries.files { - let filename = match savedfile.filename { - Some(s) => s, - None => "None".into(), - }; - let mut file = match File::open(savedfile.path) { - Ok(file) => file, - Err(error) => { - return Err(IronError::new(error, - (status::InternalServerError, - "Server couldn't save file"))) - } - }; - let mut contents = String::new(); - if let Err(error) = file.read_to_string(&mut contents) { - return Err(IronError::new(error, (status::BadRequest, "The file was not a text"))); - } + for (name, files) in entries.files { + println!("Field {:?} has {} files:", name, files.len()); - println!(r#"Field "{}" is file "{}":"#, name, filename); - println!("{}", contents); + for file in files { + try!(print_file(&file)); + } } + Ok(Response::with((status::Ok, "Multipart data is processed"))) } + +fn print_file(saved_file: &SavedFile) -> IronResult<()> { + let mut file = match File::open(&saved_file.path) { + Ok(file) => file, + Err(error) => { + return Err(IronError::new(error, + (status::InternalServerError, + "Server couldn't open saved file"))) + } + }; + + let mut contents = String::new(); + if let Err(error) = file.read_to_string(&mut contents) { + return Err(IronError::new(error, (status::BadRequest, "The file was not a text"))); + } + + println!("File {:?} ({:?}):", saved_file.filename, saved_file.content_type); + println!("{}", contents); + + Ok(()) +} diff --git a/multipart/examples/nickel.rs b/multipart/examples/nickel.rs index 0da452e48..634c27602 100644 --- a/multipart/examples/nickel.rs +++ b/multipart/examples/nickel.rs @@ -1,4 +1,3 @@ -#[macro_use] extern crate nickel; extern crate multipart; @@ -10,12 +9,12 @@ use nickel::{HttpRouter, MiddlewareResult, Nickel, Request, Response}; fn handle_multipart<'mw>(req: &mut Request, mut res: Response<'mw>) -> MiddlewareResult<'mw> { match Multipart::from_request(req) { Ok(mut multipart) => { - match multipart.save_all() { + match multipart.save().temp() { SaveResult::Full(entries) => process_entries(res, entries), SaveResult::Partial(entries, e) => { println!("Partial errors ... {:?}", e); - return process_entries(res, entries); + return process_entries(res, entries.keep_partial()); }, SaveResult::Error(e) => { @@ -36,31 +35,31 @@ fn handle_multipart<'mw>(req: &mut Request, mut res: Response<'mw>) -> Middlewar /// Returns an OK response or an error. fn process_entries<'mw>(res: Response<'mw>, entries: Entries) -> MiddlewareResult<'mw> { for (name, field) in entries.fields { - println!(r#"Field "{}": "{}""#, name, field); + println!("Field {:?}: {:?}", name, field); } - for (name, savedfile) in entries.files { - let filename = match savedfile.filename { - Some(s) => s, - None => "None".into(), - }; + for (name, files) in entries.files { + println!("Field {:?} has {} files:", name, files.len()); - match File::open(savedfile.path) { - Ok(mut file) => { - let mut contents = String::new(); - match file.read_to_string(&mut contents) { - Ok(sz) => println!("File: \"{}\" is of size: {}b.", filename, sz), - Err(e) => println!("Could not read file's \"{}\" size. Error: {:?}", filename, e), + for saved_file in files { + match File::open(&saved_file.path) { + Ok(mut file) => { + let mut contents = String::new(); + if let Err(e) = file.read_to_string(&mut contents) { + println!("Could not read file {:?}. Error: {:?}", saved_file.filename, e); + return res.error(nickel::status::StatusCode::BadRequest, "The uploaded file was not readable") + } + + println!("File {:?} ({:?}):", saved_file.filename, saved_file.content_type); + println!("{}", contents); + file } - println!(r#"Field "{}" is file "{}":"#, name, filename); - println!("{}", contents); - file - } - Err(e) => { - println!("Could open file \"{}\". Error: {:?}", filename, e); - return res.error(nickel::status::StatusCode::BadRequest, "The uploaded file was not readable") - } - }; + Err(e) => { + println!("Could open file {:?}. Error: {:?}", saved_file.filename, e); + return res.error(nickel::status::StatusCode::BadRequest, "The uploaded file was not readable") + } + }; + } } res.send("Ok") diff --git a/multipart/examples/tiny_http.rs b/multipart/examples/tiny_http.rs index 4827d75f6..0c7b4d4af 100644 --- a/multipart/examples/tiny_http.rs +++ b/multipart/examples/tiny_http.rs @@ -2,8 +2,8 @@ extern crate tiny_http; extern crate multipart; use std::fs::File; -use std::io::{Error, Read}; -use multipart::server::{Multipart, Entries, SaveResult}; +use std::io::{self, Read}; +use multipart::server::{Multipart, Entries, SaveResult, SavedFile}; use tiny_http::{Response, StatusCode, Request}; fn main() { // Starting a server on `localhost:80` @@ -28,18 +28,19 @@ fn main() { } /// Processes a request and returns response or an occured error. -fn process_request<'a, 'b>(request: &'a mut Request) -> Result, Error> { +fn process_request<'a, 'b>(request: &'a mut Request) -> io::Result> { // Getting a multipart reader wrapper match Multipart::from_request(request) { Ok(mut multipart) => { // Fetching all data and processing it. - // save_all() reads the request fully, parsing all fields and saving all files + // save().temp() reads the request fully, parsing all fields and saving all files // in a new temporary directory under the OS temporary directory. - match multipart.save_all() { + match multipart.save().temp() { SaveResult::Full(entries) => process_entries(entries), - SaveResult::Partial(entries, error) => { - try!(process_entries(entries)); - Err(error) + SaveResult::Partial(entries, reason) => { + try!(process_entries(entries.keep_partial())); + // We don't set limits + Err(reason.unwrap_err()) } SaveResult::Error(error) => Err(error), } @@ -50,24 +51,32 @@ fn process_request<'a, 'b>(request: &'a mut Request) -> Result(entries: Entries) -> Result, Error> { +fn process_entries<'a>(entries: Entries) -> io::Result> { for (name, field) in entries.fields { - println!(r#"Field "{}": "{}""#, name, field); + println!("Field {:?}: {:?}", name, field); } - for (name, savedfile) in entries.files { - let filename = match savedfile.filename { - Some(s) => s, - None => "None".into(), - }; - let mut file = try!(File::open(savedfile.path)); - let mut contents = String::new(); - try!(file.read_to_string(&mut contents)); + for (name, files) in entries.files { + println!("Field {:?} has {} files:", name, files.len()); - println!(r#"Field "{}" is file "{}":"#, name, filename); - println!("{}", contents); + for file in files { + try!(print_file(&file)); + } } - Ok(build_response(200, "Multipart data is received!".into())) + + Ok(build_response(200, "Multipart data is received!")) +} + +fn print_file(saved_file: &SavedFile) -> io::Result<()> { + let mut file = try!(File::open(&saved_file.path)); + + let mut contents = String::new(); + try!(file.read_to_string(&mut contents)); + + println!("File {:?} ({:?}):", saved_file.filename, saved_file.content_type); + println!("{}", contents); + + Ok(()) } /// A utility function to build responses using only two arguments diff --git a/multipart/src/local_test.rs b/multipart/src/local_test.rs index dd43e022b..aa1519be3 100644 --- a/multipart/src/local_test.rs +++ b/multipart/src/local_test.rs @@ -301,7 +301,7 @@ fn test_server(buf: HttpBuffer, fields: &mut TestFields) { let mut multipart = Multipart::from_request(server_buf) .unwrap_or_else(|_| panic!("Buffer should be multipart!")); - while let Some(mut field) = multipart.read_entry().unwrap() { + while let Some(mut field) = multipart.read_entry_mut().unwrap_opt() { fields.check_field(&mut field); } } diff --git a/multipart/src/server/field.rs b/multipart/src/server/field.rs index 3105d99c5..a63235e28 100644 --- a/multipart/src/server/field.rs +++ b/multipart/src/server/field.rs @@ -584,6 +584,11 @@ pub trait ReadEntry: PrivReadEntry + Sized { } ) } + + /// Equivalent to `read_entry()` but takes `&mut self` + fn read_entry_mut(&mut self) -> ReadEntryResult<&mut Self> { + ReadEntry::read_entry(self) + } } impl ReadEntry for T where T: PrivReadEntry {} diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index 4bece0005..5639f72b7 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -31,7 +31,7 @@ pub use self::field::{MultipartField, MultipartFile, MultipartData, ReadEntry, R use self::save::SaveBuilder; -pub use self::save::{Entries, SaveResult}; +pub use self::save::{Entries, SaveResult, SavedFile}; use self::save::EntriesSaveResult; @@ -116,7 +116,7 @@ impl Multipart { /// If the previously returned entry had contents of type `MultipartField::File`, /// calling this again will discard any unread contents of that entry. pub fn read_entry(&mut self) -> io::Result>> { - ReadEntry::read_entry(self).into_result() + self.read_entry_mut().into_result() } /// Read the next entry from this multipart request, returning a struct with the field's name and diff --git a/multipart/src/server/save.rs b/multipart/src/server/save.rs index 227c59f90..80dda3ba1 100644 --- a/multipart/src/server/save.rs +++ b/multipart/src/server/save.rs @@ -521,6 +521,22 @@ impl From for PartialReason { } } +impl PartialReason { + /// Return `io::Error` in the `IoError` case or panic otherwise. + pub fn unwrap_err(self) -> io::Error { + self.expect_err("`PartialReason` was not `IoError`") + } + + /// Return `io::Error` in the `IoError` case or panic with the given + /// message otherwise. + pub fn expect_err(self, msg: &str) -> io::Error { + match self { + PartialReason::IoError(e) => e, + _ => panic!("{}: {:?}", msg, self), + } + } +} + /// The file field that was being read when the save operation quit. /// /// May be partially saved to the filesystem if `dest` is `Some`. From c6e8d53cab697b33f85615e13cf1c05df9d5140a Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Tue, 28 Feb 2017 11:32:55 -0800 Subject: [PATCH 249/453] Set release version --- multipart/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 16784078b..506ae35c4 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "multipart" -version = "0.10.0-alpha.3" +version = "0.10.0" authors = ["Austin Bonander "] From 1e595eb92066b94679b7fd8d404f7fd2a49028a0 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Tue, 28 Feb 2017 11:53:33 -0800 Subject: [PATCH 250/453] Docs tweaks --- multipart/src/server/field.rs | 16 ++++++++-------- multipart/src/server/mod.rs | 18 ++++++++++-------- multipart/src/server/save.rs | 12 +++++++----- 3 files changed, 25 insertions(+), 21 deletions(-) diff --git a/multipart/src/server/field.rs b/multipart/src/server/field.rs index a63235e28..09347ea20 100644 --- a/multipart/src/server/field.rs +++ b/multipart/src/server/field.rs @@ -375,7 +375,7 @@ impl MultipartFile { /// permissions, but you should still not use user input directly as filesystem paths. /// If it is truly necessary, you should sanitize filenames such that they cannot be /// misinterpreted by the OS. Such functionality is outside the scope of this crate. - #[deprecated = "`filename` field is now public"] + #[deprecated(since = "0.10.0", note = "`filename` field is now public")] pub fn filename(&self) -> Option<&str> { self.filename.as_ref().map(String::as_ref) } @@ -390,7 +390,7 @@ impl MultipartFile { /// Some variants wrap arbitrary strings which could be abused by a malicious user if your /// application performs any non-idempotent operations based on their value, such as /// starting another program or querying/updating a database (web-search "SQL injection"). - #[deprecated = "`content_type` field is now public"] + #[deprecated(since = "0.10.0", note = "`content_type` field is now public")] pub fn content_type(&self) -> &Mime { &self.content_type } @@ -421,7 +421,7 @@ impl MultipartFile where M: ReadEntry { /// If successful, returns the number of bytes written. /// /// Retries when `io::Error::kind() == io::ErrorKind::Interrupted`. - #[deprecated = "use `.save().write_to()` instead"] + #[deprecated(since = "0.10.0", note = "use `.save().write_to()` instead")] pub fn save_to(&mut self, out: W) -> io::Result { self.save().write_to(out).into_result_strict() } @@ -432,7 +432,7 @@ impl MultipartFile where M: ReadEntry { /// If successful, returns the number of bytes written. /// /// Retries when `io::Error::kind() == io::ErrorKind::Interrupted`. - #[deprecated = "use `.save().limit(limit).write_to(out)` instead"] + #[deprecated(since = "0.10.0", note = "use `.save().size_limit(limit).write_to(out)` instead")] pub fn save_to_limited(&mut self, out: W, limit: u64) -> io::Result { self.save().size_limit(limit).write_to(out).into_result_strict() } @@ -442,7 +442,7 @@ impl MultipartFile where M: ReadEntry { /// Returns the saved file info on success, or any errors otherwise. /// /// Retries when `io::Error::kind() == io::ErrorKind::Interrupted`. - #[deprecated = "use `.save().with_path(path)` instead"] + #[deprecated(since = "0.10.0", note = "use `.save().with_path(path)` instead")] pub fn save_as>(&mut self, path: P) -> io::Result { self.save().with_path(path).into_result_strict() } @@ -455,7 +455,7 @@ impl MultipartFile where M: ReadEntry { /// Returns the saved file's info on success, or any errors otherwise. /// /// Retries when `io::Error::kind() == io::ErrorKind::Interrupted`. - #[deprecated = "use `.save().with_dir(dir)` instead"] + #[deprecated(since = "0.10.0", note = "use `.save().with_dir(dir)` instead")] pub fn save_in>(&mut self, dir: P) -> io::Result { self.save().with_dir(dir.as_ref()).into_result_strict() } @@ -467,7 +467,7 @@ impl MultipartFile where M: ReadEntry { /// Returns the saved file's info on success, or any errors otherwise. /// /// Retries when `io::Error::kind() == io::ErrorKind::Interrupted`. - #[deprecated = "use `.save().limit(limit).with_path(path)` instead"] + #[deprecated(since = "0.10.0", note = "use `.save().size_limit(limit).with_path(path)` instead")] pub fn save_as_limited>(&mut self, path: P, limit: u64) -> io::Result { self.save().size_limit(limit).with_path(path).into_result_strict() } @@ -482,7 +482,7 @@ impl MultipartFile where M: ReadEntry { /// Returns the saved file's info on success, or any errors otherwise. /// /// Retries when `io::Error::kind() == io::ErrorKind::Interrupted`. - #[deprecated = "use `.save().limit(limit).with_dir(dir)` instead"] + #[deprecated(since = "0.10.0", note = "use `.save().size_limit(limit).with_dir(dir)` instead")] pub fn save_in_limited>(&mut self, dir: P, limit: u64) -> io::Result { self.save().size_limit(limit).with_dir(dir).into_result_strict() } diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index 5639f72b7..a571bd596 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -142,6 +142,8 @@ impl Multipart { } /// Get a builder type for saving the files in this request to the filesystem. + /// + /// See [`SaveBuilder`](save/struct.SaveBuilder.html) for more information. pub fn save(&mut self) -> SaveBuilder<&mut Self> { SaveBuilder::new(self) } @@ -150,8 +152,8 @@ impl Multipart { /// directory under the OS temporary directory. /// /// If there is an error in reading the request, returns the partial result along with the - /// error. See [`SaveResult`](enum.SaveResult.html) for more information. - #[deprecated = "use `.save().temp()` instead"] + /// error. See [`SaveResult`](save/enum.SaveResult.html) for more information. + #[deprecated(since = "0.10.0", note = "use `.save().temp()` instead")] pub fn save_all(&mut self) -> EntriesSaveResult<&mut Self> { self.save().temp() } @@ -160,8 +162,8 @@ impl Multipart { /// directory under `dir`. /// /// If there is an error in reading the request, returns the partial result along with the - /// error. See [`SaveResult`](enum.SaveResult.html) for more information. - #[deprecated = "use `.save().with_temp_dir()` instead"] + /// error. See [`SaveResult`](save/enum.SaveResult.html) for more information. + #[deprecated(since = "0.10.0", note = "use `.save().with_temp_dir()` instead")] pub fn save_all_under>(&mut self, dir: P) -> EntriesSaveResult<&mut Self> { match TempDir::new_in(dir, "multipart") { Ok(temp_dir) => self.save().with_temp_dir(temp_dir), @@ -175,8 +177,8 @@ impl Multipart { /// Files larger than `limit` will be truncated to `limit`. /// /// If there is an error in reading the request, returns the partial result along with the - /// error. See [`SaveResult`](enum.SaveResult.html) for more information. - #[deprecated = "use `.save().limit(limit)` instead"] + /// error. See [`SaveResult`](save/enum.SaveResult.html) for more information. + #[deprecated(since = "0.10.0", note = "use `.save().size_limit(limit)` instead")] pub fn save_all_limited(&mut self, limit: u64) -> EntriesSaveResult<&mut Self> { self.save().size_limit(limit).temp() } @@ -187,8 +189,8 @@ impl Multipart { /// Files larger than `limit` will be truncated to `limit`. /// /// If there is an error in reading the request, returns the partial result along with the - /// error. See [`SaveResult`](enum.SaveResult.html) for more information. - #[deprecated = "use `.save().limit(limit).with_temp_dir()` instead"] + /// error. See [`SaveResult`](save/enum.SaveResult.html) for more information. + #[deprecated(since = "0.10.0", note = "use `.save().size_limit(limit).with_temp_dir()` instead")] pub fn save_all_under_limited>(&mut self, dir: P, limit: u64) -> EntriesSaveResult<&mut Self> { match TempDir::new_in(dir, "multipart") { Ok(temp_dir) => self.save().size_limit(limit).with_temp_dir(temp_dir), diff --git a/multipart/src/server/save.rs b/multipart/src/server/save.rs index 80dda3ba1..e604c4da7 100644 --- a/multipart/src/server/save.rs +++ b/multipart/src/server/save.rs @@ -49,14 +49,14 @@ macro_rules! try_start ( /// `mod_open_opts()`. /// /// ### File Size and Count Limits -/// You can set a size limit for individual files with `limit()`, which takes either `u64` +/// You can set a size limit for individual files with `size_limit()`, which takes either `u64` /// or `Option`. /// /// You can also set the maximum number of files to process with `count_limit()`, which /// takes either `u32` or `Option`. This only has an effect when using -/// `SaveBuilder`. +/// `SaveBuilder<[&mut] Multipart>`. /// -/// ### Warning: Do **not* trust user input! +/// ### Warning: Do **not** trust user input! /// It is a serious security risk to create files or directories with paths based on user input. /// A malicious user could craft a path which can be used to overwrite important files, such as /// web templates, static assets, Javascript files, database files, configuration files, etc., @@ -109,6 +109,7 @@ impl SaveBuilder { } } +/// Save API for whole multipart requests. impl SaveBuilder where M: ReadEntry { /// Set the maximum number of files to write out. /// @@ -119,7 +120,7 @@ impl SaveBuilder where M: ReadEntry { } /// Save the file fields in the request to a new temporary directory prefixed with - /// "multipart-rs" in the OS temporary directory. + /// `multipart-rs` in the OS temporary directory. /// /// For more options, create a `TempDir` yourself and pass it to `with_temp_dir()` instead. /// @@ -152,7 +153,7 @@ impl SaveBuilder where M: ReadEntry { /// Save the file fields in the request to a new permanent directory with the given path. /// - /// Any nonexistent parent directories will be created. + /// Any nonexistent directories in the path will be created. pub fn with_dir>(self, dir: P) -> EntriesSaveResult { let dir = dir.into(); @@ -239,6 +240,7 @@ impl SaveBuilder where M: ReadEntry { } } +/// Save API for individual files. impl<'m, M: 'm> SaveBuilder<&'m mut MultipartFile> where MultipartFile: BufRead { /// Save to a file with a random alphanumeric name in the OS temporary directory. From e4ccd84ffb0c13c0990a910ebff88c9efd6bef09 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sat, 4 Mar 2017 14:59:11 -0800 Subject: [PATCH 251/453] Check that `Content-Disposition: form-data` for each field --- multipart/src/server/field.rs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/multipart/src/server/field.rs b/multipart/src/server/field.rs index 09347ea20..d1e407762 100644 --- a/multipart/src/server/field.rs +++ b/multipart/src/server/field.rs @@ -131,11 +131,16 @@ impl ContentDisp { const CONT_DISP: &'static str = "Content-Disposition"; - let header = try_opt!( + let header: &StrHeader = try_opt!( find_header(headers, CONT_DISP), error!("Field headers did not contain Content-Disposition header (required)") ); + if header.val != "form-data" { + error!("Field `Content-Disposition` header was not `form-data`"); + return; + } + const NAME: &'static str = "name="; const FILENAME: &'static str = "filename="; @@ -173,8 +178,8 @@ fn parse_cont_type(headers: &[StrHeader]) -> Option { const CONTENT_TYPE: &'static str = "Content-Type"; let header = try_opt!( - find_header(headers, CONTENT_TYPE), - debug!("Content-Type header not found for field.") + find_header(headers, CONTENT_TYPE), + debug!("Content-Type header not found for field.") ); // Boundary parameter will be parsed into the `Mime` From 2a2b00b2712c348eee403fd39c5fe697eebafd3b Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sat, 4 Mar 2017 15:04:47 -0800 Subject: [PATCH 252/453] Add a way to construct a standalone `mock::ServerRequest` --- multipart/src/mock.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/multipart/src/mock.rs b/multipart/src/mock.rs index 67011209a..53a1778ed 100644 --- a/multipart/src/mock.rs +++ b/multipart/src/mock.rs @@ -134,6 +134,20 @@ pub struct ServerRequest<'a> { rng: ThreadRng, } +impl<'a> ServerRequest<'a> { + /// Create a new `ServerRequest` with the given data and boundary. + /// + /// Assumes `content_len: None` + pub fn new(data: &'a [u8], boundary: &'a str) -> Self { + ServerRequest { + data: data, + boundary: boundary, + content_len: None, + rng: rand::thread_rng(), + } + } +} + impl<'a> Read for ServerRequest<'a> { /// To simulate a network connection, this will copy a random number of bytes /// from the buffer to `out`. From 9455f6bfac639b4c1274fe2b2ee10c6ba8335ca8 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sat, 4 Mar 2017 15:05:21 -0800 Subject: [PATCH 253/453] Fixup previous change to field.rs --- multipart/src/server/field.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/src/server/field.rs b/multipart/src/server/field.rs index d1e407762..351aa9721 100644 --- a/multipart/src/server/field.rs +++ b/multipart/src/server/field.rs @@ -138,7 +138,7 @@ impl ContentDisp { if header.val != "form-data" { error!("Field `Content-Disposition` header was not `form-data`"); - return; + return None; } const NAME: &'static str = "name="; From 9286b7f75c807ab5ad45dfa8149e25f029a62c05 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sat, 4 Mar 2017 15:17:25 -0800 Subject: [PATCH 254/453] Preliminary fuzzing support --- multipart/fuzz/.gitignore | 5 + multipart/fuzz/Cargo.lock | 205 +++++++++++++++++++++++++ multipart/fuzz/Cargo.toml | 24 +++ multipart/fuzz/fuzzer_dict | 7 + multipart/fuzz/fuzzers/server_basic.rs | 36 +++++ 5 files changed, 277 insertions(+) create mode 100644 multipart/fuzz/.gitignore create mode 100644 multipart/fuzz/Cargo.lock create mode 100644 multipart/fuzz/Cargo.toml create mode 100644 multipart/fuzz/fuzzer_dict create mode 100644 multipart/fuzz/fuzzers/server_basic.rs diff --git a/multipart/fuzz/.gitignore b/multipart/fuzz/.gitignore new file mode 100644 index 000000000..dfeb7db19 --- /dev/null +++ b/multipart/fuzz/.gitignore @@ -0,0 +1,5 @@ + +target +libfuzzer +corpus +artifacts diff --git a/multipart/fuzz/Cargo.lock b/multipart/fuzz/Cargo.lock new file mode 100644 index 000000000..12f9e3da4 --- /dev/null +++ b/multipart/fuzz/Cargo.lock @@ -0,0 +1,205 @@ +[root] +name = "multipart-fuzz" +version = "0.0.1" +dependencies = [ + "libfuzzer-sys 0.1.0 (git+https://github.com/rust-fuzz/libfuzzer-sys.git)", + "multipart 0.10.0", +] + +[[package]] +name = "buf_redux" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "memchr 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)", + "safemem 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "gcc" +version = "0.3.43" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "httparse" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "libc" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "libfuzzer-sys" +version = "0.1.0" +source = "git+https://github.com/rust-fuzz/libfuzzer-sys.git#9d00b47e0ef203543d304f2a969a3ced8817369c" +dependencies = [ + "gcc 0.3.43 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "log" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "memchr" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "mime" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "mime_guess" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "mime 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "phf 0.7.21 (registry+https://github.com/rust-lang/crates.io-index)", + "phf_codegen 0.7.21 (registry+https://github.com/rust-lang/crates.io-index)", + "unicase 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "multipart" +version = "0.10.0" +dependencies = [ + "buf_redux 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", + "httparse 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", + "mime 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "mime_guess 1.8.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)", + "safemem 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "tempdir 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", + "twoway 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "phf" +version = "0.7.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "phf_shared 0.7.21 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "phf_codegen" +version = "0.7.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "phf_generator 0.7.21 (registry+https://github.com/rust-lang/crates.io-index)", + "phf_shared 0.7.21 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "phf_generator" +version = "0.7.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "phf_shared 0.7.21 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "phf_shared" +version = "0.7.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "siphasher 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "unicase 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rand" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "rustc_version" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "semver 0.1.20 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "safemem" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "safemem" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "semver" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "siphasher" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "tempdir" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rand 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "twoway" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "memchr 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "unicase" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "rustc_version 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[metadata] +"checksum buf_redux 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e1497634c131ba13483b6e8123f69e219253b018bb32949eefd55c6b5051585d" +"checksum gcc 0.3.43 (registry+https://github.com/rust-lang/crates.io-index)" = "c07c758b972368e703a562686adb39125707cc1ef3399da8c019fc6c2498a75d" +"checksum httparse 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a6e7a63e511f9edffbab707141fbb8707d1a3098615fb2adbd5769cdfcc9b17d" +"checksum libc 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)" = "88ee81885f9f04bff991e306fea7c1c60a5f0f9e409e99f6b40e3311a3363135" +"checksum libfuzzer-sys 0.1.0 (git+https://github.com/rust-fuzz/libfuzzer-sys.git)" = "" +"checksum log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "ab83497bf8bf4ed2a74259c1c802351fcd67a65baa86394b6ba73c36f4838054" +"checksum memchr 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)" = "d8b629fb514376c675b98c1421e80b151d3817ac42d7c667717d282761418d20" +"checksum mime 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "b5c93a4bd787ddc6e7833c519b73a50883deb5863d76d9b71eb8216fb7f94e66" +"checksum mime_guess 1.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "76da6df85047af8c0edfa53f48eb1073012ce1cc95c8fedc0a374f659a89dd65" +"checksum phf 0.7.21 (registry+https://github.com/rust-lang/crates.io-index)" = "cb325642290f28ee14d8c6201159949a872f220c62af6e110a56ea914fbe42fc" +"checksum phf_codegen 0.7.21 (registry+https://github.com/rust-lang/crates.io-index)" = "d62594c0bb54c464f633175d502038177e90309daf2e0158be42ed5f023ce88f" +"checksum phf_generator 0.7.21 (registry+https://github.com/rust-lang/crates.io-index)" = "6b07ffcc532ccc85e3afc45865469bf5d9e4ef5bfcf9622e3cfe80c2d275ec03" +"checksum phf_shared 0.7.21 (registry+https://github.com/rust-lang/crates.io-index)" = "07e24b0ca9643bdecd0632f2b3da6b1b89bbb0030e0b992afc1113b23a7bc2f2" +"checksum rand 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)" = "022e0636ec2519ddae48154b028864bdce4eaf7d35226ab8e65c611be97b189d" +"checksum rustc_version 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "c5f5376ea5e30ce23c03eb77cbe4962b988deead10910c372b226388b594c084" +"checksum safemem 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "725b3bf47ae40b4abcd27b5f0a9540369426a29f7b905649b3e1468e13e22009" +"checksum safemem 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e27a8b19b835f7aea908818e871f5cc3a5a186550c30773be987e155e8163d8f" +"checksum semver 0.1.20 (registry+https://github.com/rust-lang/crates.io-index)" = "d4f410fedcf71af0345d7607d246e7ad15faaadd49d240ee3b24e5dc21a820ac" +"checksum siphasher 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2ffc669b726f2bc9a3bcff66e5e23b56ba6bf70e22a34c3d7b6d0b3450b65b84" +"checksum tempdir 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "87974a6f5c1dfb344d733055601650059a3363de2a6104819293baff662132d6" +"checksum twoway 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "e267e178055eb3b081224bbef62d4f508ae3c9f000b6ae6ccdb04a0d9c34b77f" +"checksum unicase 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "13a5906ca2b98c799f4b1ab4557b76367ebd6ae5ef14930ec841c74aed5f3764" diff --git a/multipart/fuzz/Cargo.toml b/multipart/fuzz/Cargo.toml new file mode 100644 index 000000000..7807c19da --- /dev/null +++ b/multipart/fuzz/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "multipart-fuzz" +version = "0.0.1" +authors = ["Automatically generated"] +publish = false + +[package.metadata] +cargo-fuzz = true + +[dependencies.multipart] +path = ".." +default-features = "false" +features = ["mock", "client", "server"] + +[dependencies.libfuzzer-sys] +git = "https://github.com/rust-fuzz/libfuzzer-sys.git" + +# Prevent this from interfering with workspaces +[workspace] +members = ["."] + +[[bin]] +name = "server_basic" +path = "fuzzers/server_basic.rs" diff --git a/multipart/fuzz/fuzzer_dict b/multipart/fuzz/fuzzer_dict new file mode 100644 index 000000000..f44d26324 --- /dev/null +++ b/multipart/fuzz/fuzzer_dict @@ -0,0 +1,7 @@ +"Content-Disposition: form-data; name=" +# CR LF +"\x0D\x0A" +"Content-Type:" +"filename=" +"--12--34--56" +"--12--34--56--" \ No newline at end of file diff --git a/multipart/fuzz/fuzzers/server_basic.rs b/multipart/fuzz/fuzzers/server_basic.rs new file mode 100644 index 000000000..972cd2c38 --- /dev/null +++ b/multipart/fuzz/fuzzers/server_basic.rs @@ -0,0 +1,36 @@ +#![no_main] +extern crate libfuzzer_sys; +extern crate multipart; + +use multipart::server::{Multipart, MultipartData}; +use multipart::mock::ServerRequest; + +use std::io::BufRead; + +const BOUNDARY: &'static str = "--12--34--56"; + +#[export_name="rust_fuzzer_test_input"] +pub extern fn go(data: &[u8]) { + if data.len() < BOUNDARY.len() { return; } + + let req = ServerRequest::new(data, BOUNDARY); + + let mut multipart = if let Ok(multi) = Multipart::from_request(req) { + multi + } else { + panic!("This shouldn't have failed") + }; + + // A lot of requests will be malformed + while let Ok(Some(entry)) = multipart.read_entry() { + match entry.data { + MultipartData::Text(_) => (), + MultipartData::File(mut file) => loop { + let consume = file.fill_buf().expect("This shouldn't fail").len(); + + if consume == 0 { break; } + file.consume(consume); + } + } + } +} From cd64c7dd24ebb32b06dd99933f680547033f4a1e Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sat, 4 Mar 2017 15:49:01 -0800 Subject: [PATCH 255/453] Duh, we already check the `Content-Disposition` header --- multipart/src/server/field.rs | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/multipart/src/server/field.rs b/multipart/src/server/field.rs index 351aa9721..c7263480d 100644 --- a/multipart/src/server/field.rs +++ b/multipart/src/server/field.rs @@ -131,16 +131,11 @@ impl ContentDisp { const CONT_DISP: &'static str = "Content-Disposition"; - let header: &StrHeader = try_opt!( + let header = try_opt!( find_header(headers, CONT_DISP), error!("Field headers did not contain Content-Disposition header (required)") ); - if header.val != "form-data" { - error!("Field `Content-Disposition` header was not `form-data`"); - return None; - } - const NAME: &'static str = "name="; const FILENAME: &'static str = "filename="; From 40ebdc7a82a4b28711780945f88a6d3afc38cf51 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sat, 4 Mar 2017 16:45:57 -0800 Subject: [PATCH 256/453] Setup logging in fuzzer --- multipart/fuzz/Cargo.lock | 1 + multipart/fuzz/Cargo.toml | 3 +++ multipart/fuzz/fuzzers/logger.rs | 24 ++++++++++++++++++++++++ multipart/fuzz/fuzzers/server_basic.rs | 19 ++++++++++++++++++- 4 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 multipart/fuzz/fuzzers/logger.rs diff --git a/multipart/fuzz/Cargo.lock b/multipart/fuzz/Cargo.lock index 12f9e3da4..5d8ef624a 100644 --- a/multipart/fuzz/Cargo.lock +++ b/multipart/fuzz/Cargo.lock @@ -3,6 +3,7 @@ name = "multipart-fuzz" version = "0.0.1" dependencies = [ "libfuzzer-sys 0.1.0 (git+https://github.com/rust-fuzz/libfuzzer-sys.git)", + "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", "multipart 0.10.0", ] diff --git a/multipart/fuzz/Cargo.toml b/multipart/fuzz/Cargo.toml index 7807c19da..a9b22eb36 100644 --- a/multipart/fuzz/Cargo.toml +++ b/multipart/fuzz/Cargo.toml @@ -7,6 +7,9 @@ publish = false [package.metadata] cargo-fuzz = true +[dependencies] +log = "*" + [dependencies.multipart] path = ".." default-features = "false" diff --git a/multipart/fuzz/fuzzers/logger.rs b/multipart/fuzz/fuzzers/logger.rs new file mode 100644 index 000000000..f1f6a8ae2 --- /dev/null +++ b/multipart/fuzz/fuzzers/logger.rs @@ -0,0 +1,24 @@ +extern crate log; + +use self::log::{LogLevel, LogLevelFilter, Log, LogMetadata, LogRecord}; + +const MAX_LOG_LEVEL: LogLevel = LogLevel::Trace; + +struct Logger; + +impl Log for Logger { + fn enabled(&self, metadata: &LogMetadata) -> bool { + metadata.level() <= MAX_LOG_LEVEL + } + + fn log(&self, record: &LogRecord) { + println!("{}: {}", record.level(), record.args()); + } +} + +pub fn init() { + let _ = log::set_logger(|max_lvl| { + max_lvl.set(MAX_LOG_LEVEL.to_log_level_filter()); + Box::new(Logger) + }); +} diff --git a/multipart/fuzz/fuzzers/server_basic.rs b/multipart/fuzz/fuzzers/server_basic.rs index 972cd2c38..48a1d256d 100644 --- a/multipart/fuzz/fuzzers/server_basic.rs +++ b/multipart/fuzz/fuzzers/server_basic.rs @@ -2,19 +2,34 @@ extern crate libfuzzer_sys; extern crate multipart; +#[macro_use] +extern crate log; + use multipart::server::{Multipart, MultipartData}; use multipart::mock::ServerRequest; +mod logger; + use std::io::BufRead; const BOUNDARY: &'static str = "--12--34--56"; #[export_name="rust_fuzzer_test_input"] pub extern fn go(data: &[u8]) { + logger::init(); + + do_fuzz(data); +} + +fn do_fuzz(data: &[u8]) { + info!("Fuzzing started! Data len: {}", data.len()); + if data.len() < BOUNDARY.len() { return; } let req = ServerRequest::new(data, BOUNDARY); + info!("Request constructed!"); + let mut multipart = if let Ok(multi) = Multipart::from_request(req) { multi } else { @@ -28,9 +43,11 @@ pub extern fn go(data: &[u8]) { MultipartData::File(mut file) => loop { let consume = file.fill_buf().expect("This shouldn't fail").len(); + info!("Consume amt: {}", consume); + if consume == 0 { break; } file.consume(consume); } } } -} +} \ No newline at end of file From 87c7cd2c04046f2a25ed76f13101b4075b8f45bb Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sat, 4 Mar 2017 16:52:24 -0800 Subject: [PATCH 257/453] Handle zero-sized reads better --- multipart/src/server/boundary.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/multipart/src/server/boundary.rs b/multipart/src/server/boundary.rs index 9276b3886..f49182dec 100644 --- a/multipart/src/server/boundary.rs +++ b/multipart/src/server/boundary.rs @@ -49,7 +49,10 @@ impl BoundaryReader where R: Read { let buf = try!(fill_buf_min(&mut self.source, min_len)); - self.at_end = buf.len() == 0; + if buf.len() == 0 { + self.at_end = true; + return Ok(buf); + } if log_enabled!(LogLevel::Trace) { trace!("Buf: {:?}", String::from_utf8_lossy(buf)); From 0d7e364fcc7c2f2fd4769b2515b2e85e6f24fedd Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sat, 4 Mar 2017 21:56:10 -0800 Subject: [PATCH 258/453] Don't test fuzzing branch on Travis --- multipart/.travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/multipart/.travis.yml b/multipart/.travis.yml index 490a64c65..acae83dd7 100644 --- a/multipart/.travis.yml +++ b/multipart/.travis.yml @@ -1,6 +1,9 @@ language: rust cache: cargo sudo: false +branches: + except: + - fuzzing rust: - stable - beta From 0a4b0214853584a649a216d0909e44c7552bd846 Mon Sep 17 00:00:00 2001 From: Austin Date: Sat, 4 Mar 2017 22:35:27 -0800 Subject: [PATCH 259/453] Add more logging statements --- multipart/fuzz/fuzzers/server_basic.rs | 8 ++++++-- multipart/src/server/boundary.rs | 1 + multipart/src/server/field.rs | 2 ++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/multipart/fuzz/fuzzers/server_basic.rs b/multipart/fuzz/fuzzers/server_basic.rs index 48a1d256d..09312acce 100644 --- a/multipart/fuzz/fuzzers/server_basic.rs +++ b/multipart/fuzz/fuzzers/server_basic.rs @@ -18,11 +18,14 @@ const BOUNDARY: &'static str = "--12--34--56"; pub extern fn go(data: &[u8]) { logger::init(); + info!("Fuzzing started! Data len: {}", data.len()); + do_fuzz(data); + + info!("Finished fuzzing iteration"); } fn do_fuzz(data: &[u8]) { - info!("Fuzzing started! Data len: {}", data.len()); if data.len() < BOUNDARY.len() { return; } @@ -38,6 +41,7 @@ fn do_fuzz(data: &[u8]) { // A lot of requests will be malformed while let Ok(Some(entry)) = multipart.read_entry() { + info!("read_entry() loop!"); match entry.data { MultipartData::Text(_) => (), MultipartData::File(mut file) => loop { @@ -50,4 +54,4 @@ fn do_fuzz(data: &[u8]) { } } } -} \ No newline at end of file +} diff --git a/multipart/src/server/boundary.rs b/multipart/src/server/boundary.rs index f49182dec..b369557f5 100644 --- a/multipart/src/server/boundary.rs +++ b/multipart/src/server/boundary.rs @@ -50,6 +50,7 @@ impl BoundaryReader where R: Read { let buf = try!(fill_buf_min(&mut self.source, min_len)); if buf.len() == 0 { + debug!("fill_buf_min returned zero-sized buf"); self.at_end = true; return Ok(buf); } diff --git a/multipart/src/server/field.rs b/multipart/src/server/field.rs index c7263480d..dd8286473 100644 --- a/multipart/src/server/field.rs +++ b/multipart/src/server/field.rs @@ -536,6 +536,8 @@ fn find_header<'a, 'b>(headers: &'a [StrHeader<'b>], name: &str) -> Option<&'a S pub trait ReadEntry: PrivReadEntry + Sized { /// Attempt to read the next entry in the multipart stream. fn read_entry(mut self) -> ReadEntryResult { + debug!("ReadEntry::read_entry()"); + if try_read_entry!(self; self.consume_boundary()) { return End(self); } From 3e247e201d1f31923c79102db010fb38ae7b0f5a Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sat, 4 Mar 2017 22:47:54 -0800 Subject: [PATCH 260/453] Fix another infinite loop --- multipart/src/server/field.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/multipart/src/server/field.rs b/multipart/src/server/field.rs index dd8286473..82410a752 100644 --- a/multipart/src/server/field.rs +++ b/multipart/src/server/field.rs @@ -61,6 +61,8 @@ where R: BufRead, F: FnOnce(&[StrHeader]) -> Ret { loop { let buf = try!(r.fill_buf()); + if buf.len() == 0 { break; } + match try_io!(httparse::parse_headers(buf, &mut raw_headers)) { Status::Complete((consume_, raw_headers)) => { consume = consume_; From 097ed62e2e822a0df2cb5a0b3dc1262f6cd8f343 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sat, 4 Mar 2017 23:06:41 -0800 Subject: [PATCH 261/453] Don't continue find headers loop if buf is empty --- multipart/src/server/field.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/multipart/src/server/field.rs b/multipart/src/server/field.rs index 82410a752..42dfcd4d8 100644 --- a/multipart/src/server/field.rs +++ b/multipart/src/server/field.rs @@ -61,7 +61,11 @@ where R: BufRead, F: FnOnce(&[StrHeader]) -> Ret { loop { let buf = try!(r.fill_buf()); - if buf.len() == 0 { break; } + if buf.len() == 0 { + consume = 0; + header_len = 0; + break; + } match try_io!(httparse::parse_headers(buf, &mut raw_headers)) { Status::Complete((consume_, raw_headers)) => { From ca2eed57e88fc35f8495dae53648622b4a356f75 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sat, 4 Mar 2017 23:35:43 -0800 Subject: [PATCH 262/453] Put an upper bound on the number of loops in `field::with_headers()` --- multipart/src/server/field.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/multipart/src/server/field.rs b/multipart/src/server/field.rs index 42dfcd4d8..b2096e646 100644 --- a/multipart/src/server/field.rs +++ b/multipart/src/server/field.rs @@ -44,6 +44,7 @@ pub struct StrHeader<'a> { val: &'a str, } +const MAX_ATTEMPTS: usize = 5; fn with_headers(r: &mut R, f: F) -> io::Result where R: BufRead, F: FnOnce(&[StrHeader]) -> Ret { @@ -58,10 +59,13 @@ where R: BufRead, F: FnOnce(&[StrHeader]) -> Ret { { let mut raw_headers = [EMPTY_HEADER; HEADER_LEN]; + let mut attempts = 0; + loop { let buf = try!(r.fill_buf()); - if buf.len() == 0 { + if attempts == MAX_ATTEMPTS { + error!("Could not read field headers."); consume = 0; header_len = 0; break; @@ -73,7 +77,7 @@ where R: BufRead, F: FnOnce(&[StrHeader]) -> Ret { header_len = raw_headers.len(); break; }, - Status::Partial => (), + Status::Partial => attempts += 1, } } From 927c3293bc9eb0e96d4b8e8433a87c17d5af3295 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sun, 5 Mar 2017 14:03:09 -0800 Subject: [PATCH 263/453] Refactor `field::with_headers()` to not rely on broken lifetimes in `httparse::parse_headers()` (seanmonstar/httparse#34) --- multipart/src/server/field.rs | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/multipart/src/server/field.rs b/multipart/src/server/field.rs index b2096e646..f971a234b 100644 --- a/multipart/src/server/field.rs +++ b/multipart/src/server/field.rs @@ -7,7 +7,7 @@ //! `multipart` field header parsing. -use super::httparse::{self, EMPTY_HEADER, Status}; +use super::httparse::{self, EMPTY_HEADER, Header, Status}; use self::ReadEntryResult::*; @@ -46,15 +46,13 @@ pub struct StrHeader<'a> { const MAX_ATTEMPTS: usize = 5; -fn with_headers(r: &mut R, f: F) -> io::Result +fn with_headers(r: &mut R, closure: F) -> io::Result where R: BufRead, F: FnOnce(&[StrHeader]) -> Ret { const HEADER_LEN: usize = 4; // These are only written once so they don't need to be `mut` or initialized. let consume; - let header_len; - - let mut headers = [EMPTY_STR_HEADER; HEADER_LEN]; + let ret; { let mut raw_headers = [EMPTY_HEADER; HEADER_LEN]; @@ -66,34 +64,36 @@ where R: BufRead, F: FnOnce(&[StrHeader]) -> Ret { if attempts == MAX_ATTEMPTS { error!("Could not read field headers."); - consume = 0; - header_len = 0; - break; + // RFC: return an actual error instead? + return Ok(closure(&[])); } match try_io!(httparse::parse_headers(buf, &mut raw_headers)) { Status::Complete((consume_, raw_headers)) => { consume = consume_; - header_len = raw_headers.len(); + let mut headers = [EMPTY_STR_HEADER; HEADER_LEN]; + let headers = try!(copy_headers(raw_headers, &mut headers)); + debug!("Parsed headers: {:?}", headers); + ret = closure(headers); break; }, Status::Partial => attempts += 1, } } - - for (raw, header) in raw_headers.iter().take(header_len).zip(&mut headers) { - header.name = raw.name; - header.val = try!(io_str_utf8(raw.value)); - } } r.consume(consume); - let headers = &headers[..header_len]; + Ok(ret) +} - debug!("Parsed headers: {:?}", headers); +fn copy_headers<'h, 'b: 'h>(raw: &[Header<'b>], headers: &'h mut [StrHeader<'b>]) -> io::Result<&'h [StrHeader<'b>]> { + for (raw, header) in raw.iter().zip(&mut *headers) { + header.name = raw.name; + header.val = try!(io_str_utf8(raw.value)); + } - Ok(f(headers)) + Ok(&mut headers[..raw.len()]) } /// The headers that (may) appear before a `multipart/form-data` field. From 56b187ee9f1a41d096f2c8d0beeddd65ad2a571e Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sun, 5 Mar 2017 14:29:49 -0800 Subject: [PATCH 264/453] Use `set_logger_raw()` in fuzz logger so we don't anger leak-sanitizer-san --- multipart/fuzz/Cargo.toml | 2 +- multipart/fuzz/fuzzers/logger.rs | 16 ++++++++++------ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/multipart/fuzz/Cargo.toml b/multipart/fuzz/Cargo.toml index a9b22eb36..6b3efc7ed 100644 --- a/multipart/fuzz/Cargo.toml +++ b/multipart/fuzz/Cargo.toml @@ -12,7 +12,7 @@ log = "*" [dependencies.multipart] path = ".." -default-features = "false" +default-features = false features = ["mock", "client", "server"] [dependencies.libfuzzer-sys] diff --git a/multipart/fuzz/fuzzers/logger.rs b/multipart/fuzz/fuzzers/logger.rs index f1f6a8ae2..880410bf1 100644 --- a/multipart/fuzz/fuzzers/logger.rs +++ b/multipart/fuzz/fuzzers/logger.rs @@ -1,8 +1,8 @@ extern crate log; -use self::log::{LogLevel, LogLevelFilter, Log, LogMetadata, LogRecord}; +use self::log::{LogLevelFilter, Log, LogMetadata, LogRecord}; -const MAX_LOG_LEVEL: LogLevel = LogLevel::Trace; +const MAX_LOG_LEVEL: LogLevelFilter = LogLevelFilter::Error; struct Logger; @@ -16,9 +16,13 @@ impl Log for Logger { } } +static LOGGER: Logger = Logger; + pub fn init() { - let _ = log::set_logger(|max_lvl| { - max_lvl.set(MAX_LOG_LEVEL.to_log_level_filter()); - Box::new(Logger) - }); + let _ = unsafe { + log::set_logger_raw(|max_lvl| { + max_lvl.set(MAX_LOG_LEVEL); + &LOGGER + }) + }; } From e66888b5a141710e71b1f2144928399550e20ed7 Mon Sep 17 00:00:00 2001 From: Austin Date: Sun, 5 Mar 2017 15:25:58 -0800 Subject: [PATCH 265/453] Add fuzzing script, raise logging level --- multipart/fuzz/fuzzers/logger.rs | 2 +- multipart/fuzz_server.sh | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100755 multipart/fuzz_server.sh diff --git a/multipart/fuzz/fuzzers/logger.rs b/multipart/fuzz/fuzzers/logger.rs index 880410bf1..88d2c3909 100644 --- a/multipart/fuzz/fuzzers/logger.rs +++ b/multipart/fuzz/fuzzers/logger.rs @@ -2,7 +2,7 @@ extern crate log; use self::log::{LogLevelFilter, Log, LogMetadata, LogRecord}; -const MAX_LOG_LEVEL: LogLevelFilter = LogLevelFilter::Error; +const MAX_LOG_LEVEL: LogLevelFilter = LogLevelFilter::Debug; struct Logger; diff --git a/multipart/fuzz_server.sh b/multipart/fuzz_server.sh new file mode 100755 index 000000000..9d99a0a91 --- /dev/null +++ b/multipart/fuzz_server.sh @@ -0,0 +1,2 @@ +#! /bin/sh +cargo fuzz run server_basic -- -dict=fuzzer_dict -only_ascii=1 -timeout=60 $@ From d1eaaf888df6910847c36e5c729e16eeb02e1dca Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sun, 5 Mar 2017 18:13:46 -0800 Subject: [PATCH 266/453] Log buffer when httparse returns an error --- multipart/src/server/field.rs | 46 +++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/multipart/src/server/field.rs b/multipart/src/server/field.rs index f971a234b..429cb8e3a 100644 --- a/multipart/src/server/field.rs +++ b/multipart/src/server/field.rs @@ -54,32 +54,36 @@ where R: BufRead, F: FnOnce(&[StrHeader]) -> Ret { let consume; let ret; - { + let mut attempts = 0; + + loop { let mut raw_headers = [EMPTY_HEADER; HEADER_LEN]; - let mut attempts = 0; + let buf = try!(r.fill_buf()); - loop { - let buf = try!(r.fill_buf()); + if attempts == MAX_ATTEMPTS { + error!("Could not read field headers."); + // RFC: return an actual error instead? + return Ok(closure(&[])); + } - if attempts == MAX_ATTEMPTS { - error!("Could not read field headers."); - // RFC: return an actual error instead? - return Ok(closure(&[])); - } + // FIXME: https://github.com/seanmonstar/httparse/issues/34 + let err = match httparse::parse_headers(buf, &mut raw_headers) { + Ok(Status::Complete((consume_, raw_headers))) => { + consume = consume_; + let mut headers = [EMPTY_STR_HEADER; HEADER_LEN]; + let headers = try!(copy_headers(raw_headers, &mut headers)); + debug!("Parsed headers: {:?}", headers); + ret = closure(headers); + break; + }, + Ok(Status::Partial) => { attempts += 1; continue }, + Err(err) => err, + }; - match try_io!(httparse::parse_headers(buf, &mut raw_headers)) { - Status::Complete((consume_, raw_headers)) => { - consume = consume_; - let mut headers = [EMPTY_STR_HEADER; HEADER_LEN]; - let headers = try!(copy_headers(raw_headers, &mut headers)); - debug!("Parsed headers: {:?}", headers); - ret = closure(headers); - break; - }, - Status::Partial => attempts += 1, - } - } + error!("Error returned from parse_headers(): {}, Buf: {:?}", + err, String::from_utf8_lossy(buf)); + return Err(Error::new(ErrorKind::InvalidData, e)); } r.consume(consume); From aa615b9e46e14e119a587d80811e2363abcb5f84 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sun, 5 Mar 2017 18:15:03 -0800 Subject: [PATCH 267/453] Enable logging in Travis --- multipart/.travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/multipart/.travis.yml b/multipart/.travis.yml index acae83dd7..7093210ea 100644 --- a/multipart/.travis.yml +++ b/multipart/.travis.yml @@ -11,6 +11,9 @@ rust: os: - linux - osx +env: + # Get some logging in tests without being too verbose + - RUST_LOG=multipart=info script: - cargo build -v --features all - cargo test -v --features all From f6ea290c6a6a2381a48e52b557c4ce4a32a73699 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sun, 5 Mar 2017 18:15:43 -0800 Subject: [PATCH 268/453] Fix compile error --- multipart/src/server/field.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/src/server/field.rs b/multipart/src/server/field.rs index 429cb8e3a..d8a1aa672 100644 --- a/multipart/src/server/field.rs +++ b/multipart/src/server/field.rs @@ -83,7 +83,7 @@ where R: BufRead, F: FnOnce(&[StrHeader]) -> Ret { error!("Error returned from parse_headers(): {}, Buf: {:?}", err, String::from_utf8_lossy(buf)); - return Err(Error::new(ErrorKind::InvalidData, e)); + return Err(io::Error::new(io::ErrorKind::InvalidData, err)); } r.consume(consume); From b1b35726f4c105be622c0a233699810957926911 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sun, 5 Mar 2017 18:37:21 -0800 Subject: [PATCH 269/453] Log the boundary in server --- multipart/src/server/mod.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index a571bd596..194a0408b 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -104,8 +104,12 @@ impl Multipart<()> { impl Multipart { /// Construct a new `Multipart` with the given body reader and boundary. pub fn with_body>(body: R, boundary: Bnd) -> Self { + let boundary = boundary.into(); + + info!("Multipart::with_boundary(_, {:?}", boundary); + Multipart { - reader: BoundaryReader::from_reader(body, boundary.into()), + reader: BoundaryReader::from_reader(body, boundary), } } From 9bf5916401778f57f3e8fb576ac3045757f6e6b4 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sun, 5 Mar 2017 18:42:36 -0800 Subject: [PATCH 270/453] Add perpetual test --- multipart/src/local_test.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/multipart/src/local_test.rs b/multipart/src/local_test.rs index aa1519be3..7a0ce3b4e 100644 --- a/multipart/src/local_test.rs +++ b/multipart/src/local_test.rs @@ -178,6 +178,14 @@ macro_rules! do_test ( ); ); +#[test] +#[ignore] +fn perpetual_test_reg_reg() { + loop { + do_test!(test_client, test_server); + } +} + #[test] fn reg_client_reg_server() { do_test!(test_client, test_server); From 102ba4128661a68aa6b910516edd2d7740b74940 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sun, 5 Mar 2017 22:37:44 -0800 Subject: [PATCH 271/453] Add a number of retries to `boundary::fill_buf_min()` --- multipart/src/server/boundary.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/multipart/src/server/boundary.rs b/multipart/src/server/boundary.rs index b369557f5..ff1b98a99 100644 --- a/multipart/src/server/boundary.rs +++ b/multipart/src/server/boundary.rs @@ -205,8 +205,13 @@ impl BufRead for BoundaryReader where R: Read { } fn fill_buf_min(buf: &mut BufReader, min: usize) -> io::Result<&[u8]> { - if buf.available() < min { - try!(buf.read_into_buf()); + const MAX_ATTEMPTS: usize = 3; + + let mut attempts = 0; + + while buf.available() < min && attempts < MAX_ATTEMPTS { + if try!(buf.read_into_buf()) == 0 { break; }; + attempts += 1; } Ok(buf.get_buf()) From 2a1f7ecde55567fabbd932bf358db7c06d0a5c04 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sun, 5 Mar 2017 22:38:11 -0800 Subject: [PATCH 272/453] Add extended tests with timeouts (ignored by default) --- multipart/src/local_test.rs | 57 +++++++++++++++++++++++++++++++------ 1 file changed, 49 insertions(+), 8 deletions(-) diff --git a/multipart/src/local_test.rs b/multipart/src/local_test.rs index 7a0ce3b4e..5407619fd 100644 --- a/multipart/src/local_test.rs +++ b/multipart/src/local_test.rs @@ -178,14 +178,6 @@ macro_rules! do_test ( ); ); -#[test] -#[ignore] -fn perpetual_test_reg_reg() { - loop { - do_test!(test_client, test_server); - } -} - #[test] fn reg_client_reg_server() { do_test!(test_client, test_server); @@ -206,6 +198,55 @@ fn lazy_client_entry_server() { do_test!(test_client_lazy, test_server_entry_api); } +mod extended { + use super::*; + + use std::time::Instant; + + const TIME_LIMIT_SECS: u64 = 300; + + #[test] + #[ignore] + fn reg_client_reg_server() { + let started = Instant::now(); + + while started.elapsed().as_secs() < TIME_LIMIT_SECS { + do_test!(test_client, test_server); + } + } + + #[test] + #[ignore] + fn reg_client_entry_server() { + let started = Instant::now(); + + while started.elapsed().as_secs() < TIME_LIMIT_SECS { + do_test!(test_client, test_server_entry_api); + } + } + + #[test] + #[ignore] + fn lazy_client_reg_server() { + let started = Instant::now(); + + while started.elapsed().as_secs() < TIME_LIMIT_SECS { + do_test!(test_client_lazy, test_server); + } + } + + #[test] + #[ignore] + fn lazy_client_entry_server() { + let started = Instant::now(); + + while started.elapsed().as_secs() < TIME_LIMIT_SECS { + do_test!(test_client_lazy, test_server_entry_api); + } + } +} + + fn gen_bool() -> bool { rand::thread_rng().gen() } From d5cb1f701263a60ec8f284ac200f086fdcc03490 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sun, 5 Mar 2017 23:11:27 -0800 Subject: [PATCH 273/453] Add extended tests with timeouts (ignored by default) --- multipart/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 506ae35c4..6361754bc 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "multipart" -version = "0.10.0" +version = "0.10.1" authors = ["Austin Bonander "] From df13bcffef74c1c6b3d43bd7c72b189b4039bbe3 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Fri, 10 Mar 2017 15:03:49 -0800 Subject: [PATCH 274/453] Update Hyper to a version range `>=0.9, <0.11`, reformat Iron's version range for readability; bump patch version --- multipart/Cargo.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 6361754bc..4845efd60 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "multipart" -version = "0.10.1" +version = "0.10.2" authors = ["Austin Bonander "] @@ -29,8 +29,8 @@ safemem = { version = "0.2", optional = true } twoway = { version = "0.1", optional = true } # Optional Integrations -hyper = { version = "0.9", optional = true, default-features = false } -iron = { version = ">=0.4,<0.6", optional = true } +hyper = { version = ">=0.9, <0.11", optional = true, default-features = false } +iron = { version = ">=0.4, <0.6", optional = true } # NOTE: use `nickel_` feature, as `hyper` feature is required. nickel = { optional = true, version = "0.9" } From 8a38dc5f518959181b6abb39f08c65bdc22a961e Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Fri, 10 Mar 2017 15:09:24 -0800 Subject: [PATCH 275/453] Revert "Update Hyper to a version range `>=0.9, <0.11`, reformat Iron's version range for readability; bump patch version" Nickel needs to upgrade This reverts commit df13bcffef74c1c6b3d43bd7c72b189b4039bbe3. --- multipart/Cargo.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 4845efd60..6361754bc 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "multipart" -version = "0.10.2" +version = "0.10.1" authors = ["Austin Bonander "] @@ -29,8 +29,8 @@ safemem = { version = "0.2", optional = true } twoway = { version = "0.1", optional = true } # Optional Integrations -hyper = { version = ">=0.9, <0.11", optional = true, default-features = false } -iron = { version = ">=0.4, <0.6", optional = true } +hyper = { version = "0.9", optional = true, default-features = false } +iron = { version = ">=0.4,<0.6", optional = true } # NOTE: use `nickel_` feature, as `hyper` feature is required. nickel = { optional = true, version = "0.9" } From 21dd72c09e5e8d4c818bdac02825eb5f92754ffa Mon Sep 17 00:00:00 2001 From: Corentin Henry Date: Wed, 15 Mar 2017 18:19:38 -0700 Subject: [PATCH 276/453] use clippy --- multipart/Cargo.toml | 1 + multipart/src/lib.rs | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 6361754bc..4d6b24fb0 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -21,6 +21,7 @@ mime = "0.2" mime_guess = "1.8" rand = "0.3" tempdir = ">=0.3.4" +clippy = { version = ">=0.0, <0.1", optional = true} #Server Dependencies buf_redux = { version = "0.6", optional = true } diff --git a/multipart/src/lib.rs b/multipart/src/lib.rs index b039d7552..e5b94d290 100644 --- a/multipart/src/lib.rs +++ b/multipart/src/lib.rs @@ -28,6 +28,9 @@ //! * `nickel_`: Integration with the [Nickel](http://nickel.rs) web application framework. //! See the [`server::nickel`](server/nickel/index.html) module for more information. Enables the `hyper` //! feature. +#![cfg_attr(feature="clippy", feature(plugin))] +#![cfg_attr(feature="clippy", plugin(clippy))] +#![cfg_attr(feature="clippy", deny(clippy))] #![deny(missing_docs)] #[macro_use] @@ -105,4 +108,4 @@ mod local_test; fn random_alphanumeric(len: usize) -> String { rand::thread_rng().gen_ascii_chars().take(len).collect() -} \ No newline at end of file +} From 85421db058d6f69a7ed237182c076f9b82ef09ea Mon Sep 17 00:00:00 2001 From: Corentin Henry Date: Wed, 15 Mar 2017 18:20:39 -0700 Subject: [PATCH 277/453] clippy: `needless_return` lint --- multipart/src/server/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index 194a0408b..1c209f621 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -171,7 +171,7 @@ impl Multipart { pub fn save_all_under>(&mut self, dir: P) -> EntriesSaveResult<&mut Self> { match TempDir::new_in(dir, "multipart") { Ok(temp_dir) => self.save().with_temp_dir(temp_dir), - Err(err) => return SaveResult::Error(err), + Err(err) => SaveResult::Error(err), } } @@ -198,7 +198,7 @@ impl Multipart { pub fn save_all_under_limited>(&mut self, dir: P, limit: u64) -> EntriesSaveResult<&mut Self> { match TempDir::new_in(dir, "multipart") { Ok(temp_dir) => self.save().size_limit(limit).with_temp_dir(temp_dir), - Err(err) => return SaveResult::Error(err), + Err(err) => SaveResult::Error(err), } } } From 6efe5290833182dfd00448c70d97a15bbd289643 Mon Sep 17 00:00:00 2001 From: Corentin Henry Date: Wed, 15 Mar 2017 18:21:45 -0700 Subject: [PATCH 278/453] clippy: `double_parens` lint --- multipart/src/server/save.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/src/server/save.rs b/multipart/src/server/save.rs index e604c4da7..a1c413d99 100644 --- a/multipart/src/server/save.rs +++ b/multipart/src/server/save.rs @@ -336,7 +336,7 @@ impl<'m, M: 'm> SaveBuilder<&'m mut MultipartFile> where MultipartFile: Bu match self.savable.fill_buf() { Ok(ref buf) if buf.is_empty() => Full(copied), Ok(_) => Partial(copied, PartialReason::SizeLimit), - Err(e) => Partial(copied, PartialReason::IoError((e))) + Err(e) => Partial(copied, PartialReason::IoError(e)) } } else { try_copy_buf(&mut self.savable, &mut dest) From 22462b80043d4fb6dbb87b5cef84a581ea3d13a3 Mon Sep 17 00:00:00 2001 From: Corentin Henry Date: Wed, 15 Mar 2017 18:23:30 -0700 Subject: [PATCH 279/453] clippy: `let_and_return` lint --- multipart/src/client/mod.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/multipart/src/client/mod.rs b/multipart/src/client/mod.rs index 5960af399..d7f963cae 100644 --- a/multipart/src/client/mod.rs +++ b/multipart/src/client/mod.rs @@ -238,7 +238,7 @@ impl<'a, W: Write> MultipartWriter<'a, W> { fn write_field_headers(&mut self, name: &str, filename: Option<&str>, content_type: Option) -> io::Result<()> { - let res = chain_result! { + chain_result! { // Write the first boundary, or the boundary for the previous field. self.write_boundary(), { self.data_written = true; Ok(()) }, @@ -248,9 +248,7 @@ impl<'a, W: Write> MultipartWriter<'a, W> { content_type.map(|content_type| write!(self.inner, "\r\nContent-Type: {}", content_type)) .unwrap_or(Ok(())), self.inner.write_all(b"\r\n\r\n") - }; - - res + } } fn inner_mut(&mut self) -> &mut W { From 506f038df00afa40ccaf45ed1c40dbed1271da13 Mon Sep 17 00:00:00 2001 From: Corentin Henry Date: Wed, 15 Mar 2017 18:25:42 -0700 Subject: [PATCH 280/453] clippy: `doc_markdown` lint --- multipart/src/server/hyper.rs | 2 +- multipart/src/server/tiny_http.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/multipart/src/server/hyper.rs b/multipart/src/server/hyper.rs index b9c8a7d7e..4299b144b 100644 --- a/multipart/src/server/hyper.rs +++ b/multipart/src/server/hyper.rs @@ -7,7 +7,7 @@ //! Server-side integration with [Hyper](https://github.com/hyperium/hyper). //! Enabled with the `hyper` feature (on by default). //! -//! Also contains an implementation of [`HttpRequest`](../trait.HttpRequest.html)` +//! Also contains an implementation of [`HttpRequest`](../trait.HttpRequest.html) //! for `hyper::server::Request` and `&mut hyper::server::Request`. use hyper::net::Fresh; use hyper::header::ContentType; diff --git a/multipart/src/server/tiny_http.rs b/multipart/src/server/tiny_http.rs index 03c2c6f6c..175c5bc40 100644 --- a/multipart/src/server/tiny_http.rs +++ b/multipart/src/server/tiny_http.rs @@ -1,4 +1,4 @@ -//! Integration with [tiny_http](https://github.com/frewsxcv/tiny-http) with the `tiny_http` +//! Integration with [`tiny_http`](https://github.com/frewsxcv/tiny-http) with the `tiny_http` //! feature (optional). //! //! Contains `impl `[`HttpRequest`](../trait.HttpRequest.html)` for tiny_http::Request` (not shown From 1be8356f421d97f6fb621f5517c1e284e801abca Mon Sep 17 00:00:00 2001 From: Corentin Henry Date: Wed, 15 Mar 2017 23:14:15 -0700 Subject: [PATCH 281/453] clippy: `match_same_arms` lint --- multipart/src/server/save.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/multipart/src/server/save.rs b/multipart/src/server/save.rs index a1c413d99..7cec097b4 100644 --- a/multipart/src/server/save.rs +++ b/multipart/src/server/save.rs @@ -669,9 +669,8 @@ impl SaveResult where P: Into { pub fn into_result_strict(self) -> io::Result { match self { Full(entries) => Ok(entries), - Partial(_, PartialReason::IoError(e)) => Err(e), + Partial(_, PartialReason::IoError(e)) | Error(e) => Err(e), Partial(partial, _) => Ok(partial.into()), - Error(e) => Err(e), } } } From befd0c3a9f397219575befb74c4c39e953bd3370 Mon Sep 17 00:00:00 2001 From: Corentin Henry Date: Thu, 16 Mar 2017 07:11:27 -0700 Subject: [PATCH 282/453] clippy: `needless borrow` lint --- multipart/src/mock.rs | 4 ++-- multipart/src/server/save.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/multipart/src/mock.rs b/multipart/src/mock.rs index 53a1778ed..2b936fa95 100644 --- a/multipart/src/mock.rs +++ b/multipart/src/mock.rs @@ -167,9 +167,9 @@ impl<'a> Read for ServerRequest<'a> { impl<'a> ::server::HttpRequest for ServerRequest<'a> { type Body = Self; - fn multipart_boundary(&self) -> Option<&str> { Some(&self.boundary) } + fn multipart_boundary(&self) -> Option<&str> { Some(self.boundary) } fn body(self) -> Self::Body { self } -} \ No newline at end of file +} diff --git a/multipart/src/server/save.rs b/multipart/src/server/save.rs index 7cec097b4..3b7666151 100644 --- a/multipart/src/server/save.rs +++ b/multipart/src/server/save.rs @@ -334,7 +334,7 @@ impl<'m, M: 'm> SaveBuilder<&'m mut MultipartFile> where MultipartFile: Bu // If there's more data to be read, the field was truncated match self.savable.fill_buf() { - Ok(ref buf) if buf.is_empty() => Full(copied), + Ok(buf) if buf.is_empty() => Full(copied), Ok(_) => Partial(copied, PartialReason::SizeLimit), Err(e) => Partial(copied, PartialReason::IoError(e)) } From 28466a7e739a286f14552694c4ac5455820db6ac Mon Sep 17 00:00:00 2001 From: Corentin Henry Date: Thu, 16 Mar 2017 07:16:43 -0700 Subject: [PATCH 283/453] clippy: `len_zero` lint --- multipart/src/client/lazy.rs | 2 +- multipart/src/mock.rs | 4 ++-- multipart/src/server/boundary.rs | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/multipart/src/client/lazy.rs b/multipart/src/client/lazy.rs index c60af8cc6..dcc135364 100644 --- a/multipart/src/client/lazy.rs +++ b/multipart/src/client/lazy.rs @@ -302,7 +302,7 @@ impl<'d> PreparedFields<'d> { impl<'d> Read for PreparedFields<'d> { fn read(&mut self, buf: &mut [u8]) -> io::Result { - if buf.len() == 0 { + if buf.is_empty() { debug!("PreparedFields::read() was passed a zero-sized buffer."); return Ok(0); } diff --git a/multipart/src/mock.rs b/multipart/src/mock.rs index 2b936fa95..538a723dd 100644 --- a/multipart/src/mock.rs +++ b/multipart/src/mock.rs @@ -85,7 +85,7 @@ impl Write for HttpBuffer { /// To simulate a network connection, this will copy a random number of bytes /// from `buf` to the buffer. fn write(&mut self, buf: &[u8]) -> io::Result { - if buf.len() == 0 { + if buf.is_empty() { debug!("HttpBuffer::write() was passed a zero-sized buffer."); return Ok(0); } @@ -152,7 +152,7 @@ impl<'a> Read for ServerRequest<'a> { /// To simulate a network connection, this will copy a random number of bytes /// from the buffer to `out`. fn read(&mut self, out: &mut [u8]) -> io::Result { - if out.len() == 0 { + if out.is_empty() { debug!("ServerRequest::read() was passed a zero-sized buffer."); return Ok(0); } diff --git a/multipart/src/server/boundary.rs b/multipart/src/server/boundary.rs index ff1b98a99..52ee2a4de 100644 --- a/multipart/src/server/boundary.rs +++ b/multipart/src/server/boundary.rs @@ -49,7 +49,7 @@ impl BoundaryReader where R: Read { let buf = try!(fill_buf_min(&mut self.source, min_len)); - if buf.len() == 0 { + if buf.is_empty() { debug!("fill_buf_min returned zero-sized buf"); self.at_end = true; return Ok(buf); From 071c2badfc83aafe9d09c3a48ceb1e355bd78293 Mon Sep 17 00:00:00 2001 From: Corentin Henry Date: Thu, 16 Mar 2017 07:17:29 -0700 Subject: [PATCH 284/453] clippy: `if_let_redundant_pattern_matching` lint --- multipart/src/client/lazy.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/src/client/lazy.rs b/multipart/src/client/lazy.rs index dcc135364..ea2776240 100644 --- a/multipart/src/client/lazy.rs +++ b/multipart/src/client/lazy.rs @@ -310,7 +310,7 @@ impl<'d> Read for PreparedFields<'d> { let mut total_read = 0; while total_read < buf.len() { - if let None = self.next_field { + if self.next_field.is_none() { self.next_field = self.fields.next(); } From 5970f0540261e05e02f6a6a3be05db14ff6afd15 Mon Sep 17 00:00:00 2001 From: Corentin Henry Date: Thu, 16 Mar 2017 07:19:41 -0700 Subject: [PATCH 285/453] clippy: silence `wrong_self_convention` lint --- multipart/src/client/lazy.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/multipart/src/client/lazy.rs b/multipart/src/client/lazy.rs index ea2776240..8988aa194 100644 --- a/multipart/src/client/lazy.rs +++ b/multipart/src/client/lazy.rs @@ -507,6 +507,7 @@ mod hyper { impl<'d> super::PreparedFields<'d> { /// #### Feature: `hyper` /// Convert `self` to `hyper::client::Body`. + #[cfg_attr(feature="clippy", allow(wrong_self_convention))] pub fn to_body<'b>(&'b mut self) -> Body<'b> where 'd: 'b { if let Some(content_len) = self.content_len { Body::SizedBody(self, content_len) From 18cbad6537bc38c16be73bbc6e22ee3312626264 Mon Sep 17 00:00:00 2001 From: Corentin Henry Date: Wed, 15 Mar 2017 23:11:49 -0700 Subject: [PATCH 286/453] return an error when failing to parse headers --- multipart/src/server/field.rs | 159 ++++++++++++++++++++-------------- 1 file changed, 93 insertions(+), 66 deletions(-) diff --git a/multipart/src/server/field.rs b/multipart/src/server/field.rs index d8a1aa672..61b5e9b99 100644 --- a/multipart/src/server/field.rs +++ b/multipart/src/server/field.rs @@ -18,7 +18,8 @@ use mime::{TopLevel, Mime}; use std::io::{self, Read, BufRead, Write}; use std::ops::Deref; use std::path::{Path, PathBuf}; -use std::str; +use std::{str, fmt}; +use std::error::Error; macro_rules! try_io( ($try:expr) => ( @@ -46,7 +47,7 @@ pub struct StrHeader<'a> { const MAX_ATTEMPTS: usize = 5; -fn with_headers(r: &mut R, closure: F) -> io::Result +fn with_headers(r: &mut R, closure: F) -> Result where R: BufRead, F: FnOnce(&[StrHeader]) -> Ret { const HEADER_LEN: usize = 4; @@ -62,32 +63,28 @@ where R: BufRead, F: FnOnce(&[StrHeader]) -> Ret { let buf = try!(r.fill_buf()); if attempts == MAX_ATTEMPTS { - error!("Could not read field headers."); - // RFC: return an actual error instead? - return Ok(closure(&[])); + return Err(ParseHeaderError::Other("Unknown error while parsing headers".to_string())); } // FIXME: https://github.com/seanmonstar/httparse/issues/34 - let err = match httparse::parse_headers(buf, &mut raw_headers) { + match httparse::parse_headers(buf, &mut raw_headers) { Ok(Status::Complete((consume_, raw_headers))) => { consume = consume_; let mut headers = [EMPTY_STR_HEADER; HEADER_LEN]; - let headers = try!(copy_headers(raw_headers, &mut headers)); + let headers = copy_headers(raw_headers, &mut headers)?; debug!("Parsed headers: {:?}", headers); ret = closure(headers); break; }, - Ok(Status::Partial) => { attempts += 1; continue }, - Err(err) => err, + Ok(Status::Partial) => { + attempts += 1; + continue; + }, + Err(err) => return Err(ParseHeaderError::from(err)), }; - - error!("Error returned from parse_headers(): {}, Buf: {:?}", - err, String::from_utf8_lossy(buf)); - return Err(io::Error::new(io::ErrorKind::InvalidData, err)); } r.consume(consume); - Ok(ret) } @@ -97,7 +94,7 @@ fn copy_headers<'h, 'b: 'h>(raw: &[Header<'b>], headers: &'h mut [StrHeader<'b>] header.val = try!(io_str_utf8(raw.value)); } - Ok(&mut headers[..raw.len()]) + Ok(&headers[..raw.len()]) } /// The headers that (may) appear before a `multipart/form-data` field. @@ -110,19 +107,21 @@ pub struct FieldHeaders { impl FieldHeaders { /// Parse the field headers from the passed `BufRead`, consuming the relevant bytes. - fn read_from(r: &mut R) -> io::Result> { - with_headers(r, Self::parse) - } - - fn parse(headers: &[StrHeader]) -> Option { - let cont_disp = try_opt!( - ContentDisp::parse(headers), - debug!("Failed to read Content-Disposition") - ); - - let cont_type = parse_cont_type(headers); - - Some(FieldHeaders { + fn read_from(r: &mut R) -> Result { + with_headers(r, Self::parse)? + } + + fn parse(headers: &[StrHeader]) -> Result { + let cont_disp = ContentDisp::parse(headers)?; + let cont_type = match parse_cont_type(headers) { + Ok(cont_type) => Some(cont_type), + // The Content-Type header is not mandatory so that's ok if it was not found. + Err(ParseHeaderError::Missing) => None, + // Any other error means the header was found but we failed to parse it. This is a read + // error. + Err(e) => return Err(e), + }; + Ok(FieldHeaders { cont_disp: cont_disp, cont_type: cont_type, }) @@ -138,63 +137,42 @@ pub struct ContentDisp { } impl ContentDisp { - fn parse(headers: &[StrHeader]) -> Option { + fn parse(headers: &[StrHeader]) -> Result { if headers.is_empty() { - return None; + return Err(ParseHeaderError::Missing); } const CONT_DISP: &'static str = "Content-Disposition"; - - let header = try_opt!( - find_header(headers, CONT_DISP), - error!("Field headers did not contain Content-Disposition header (required)") - ); + let header = find_header(headers, CONT_DISP).ok_or(ParseHeaderError::Missing)?; const NAME: &'static str = "name="; const FILENAME: &'static str = "filename="; let after_disp_type = { - let (disp_type, after_disp_type) = try_opt!( - split_once(header.val, ';'), - error!("Expected additional data after Content-Disposition type, got {:?}", - header.val) - ); - - + let (disp_type, after_disp_type) = split_once(header.val, ';').ok_or(ParseHeaderError::Invalid)?; if disp_type.trim() != "form-data" { - error!("Unexpected Content-Disposition value: {:?}", disp_type); - return None; - }; - + return Err(ParseHeaderError::Invalid); + } after_disp_type }; - let (field_name, after_field_name) = try_opt!( - get_str_after(NAME, ';', after_disp_type), - error!("Expected field name and maybe filename, got {:?}", after_disp_type) - ); - + let (field_name, after_field_name) = get_str_after(NAME, ';', after_disp_type).ok_or(ParseHeaderError::Invalid)?; let field_name = trim_quotes(field_name); - let filename = get_str_after(FILENAME, ';', after_field_name) .map(|(filename, _)| trim_quotes(filename).to_owned()); - Some(ContentDisp { field_name: field_name.to_owned(), filename: filename }) + Ok(ContentDisp { field_name: field_name.to_owned(), filename: filename }) } } -fn parse_cont_type(headers: &[StrHeader]) -> Option { +fn parse_cont_type(headers: &[StrHeader]) -> Result { const CONTENT_TYPE: &'static str = "Content-Type"; - let header = try_opt!( - find_header(headers, CONTENT_TYPE), - debug!("Content-Type header not found for field.") - ); + let header = find_header(headers, CONTENT_TYPE).ok_or(ParseHeaderError::Missing)?; // Boundary parameter will be parsed into the `Mime` debug!("Found Content-Type: {:?}", header.val); - let content_type = read_content_type(header.val.trim()); - Some(content_type) + Ok(read_content_type(header.val.trim())) } /// A field in a multipart request. May be either text or a binary stream (file). @@ -556,10 +534,7 @@ pub trait ReadEntry: PrivReadEntry + Sized { return End(self); } - let field_headers = match try_read_entry!(self; self.read_headers()) { - Some(headers) => headers, - None => return End(self), - }; + let field_headers = try_read_entry!(self; self.read_headers().map_err(|e| io::Error::new(io::ErrorKind::Other, e))); let data = match field_headers.cont_type { Some(cont_type) => { @@ -619,7 +594,7 @@ pub trait PrivReadEntry { /// Returns `true` if the last boundary was read, `false` otherwise. fn consume_boundary(&mut self) -> io::Result; - fn read_headers(&mut self) -> io::Result> { + fn read_headers(&mut self) -> Result { FieldHeaders::read_from(&mut self.source()) } @@ -714,4 +689,56 @@ impl ReadEntryResult { io::Error::new(io::ErrorKind::InvalidData, msg), ) } -} \ No newline at end of file +} + +#[derive(Debug)] +pub enum ParseHeaderError { + /// The header was not found + Missing, + /// The header was found but could not be parsed + Invalid, + /// IO error + Io(io::Error), + Other(String), +} + +impl fmt::Display for ParseHeaderError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match *self { + ParseHeaderError::Missing => write!(f, "header not found (ParseHeaderError::Missing)"), + ParseHeaderError::Invalid => write!(f, "invalid header (ParseHeaderError::Invalid)"), + ParseHeaderError::Io(_) => write!(f, "could not read header (ParseHeaderError::Io)"), + ParseHeaderError::Other(ref reason) => write!(f, "unknown parsing error (ParseHeaderError::Other(\"{}\"))", reason), + } + } +} + +impl Error for ParseHeaderError { + fn description(&self) -> &str { + match *self { + ParseHeaderError::Missing => "header not found", + ParseHeaderError::Invalid => "the header is not formatted correctly", + ParseHeaderError::Io(_) => "failed to read the header", + ParseHeaderError::Other(_) => "unknown parsing error", + } + } + + fn cause(&self) -> Option<&Error> { + match *self { + ParseHeaderError::Io(ref e) => Some(e), + _ => None, + } + } +} + +impl From for ParseHeaderError { + fn from(err: io::Error) -> ParseHeaderError { + ParseHeaderError::Io(err) + } +} + +impl From for ParseHeaderError { + fn from(_err: httparse::Error) -> ParseHeaderError { + ParseHeaderError::Invalid + } +} From f5d4fe5611464ae7470ae852a3d8f7403b8d8795 Mon Sep 17 00:00:00 2001 From: Corentin Henry Date: Thu, 16 Mar 2017 07:31:22 -0700 Subject: [PATCH 287/453] remove unused `try_io!` macro --- multipart/src/server/field.rs | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/multipart/src/server/field.rs b/multipart/src/server/field.rs index 61b5e9b99..e79d656e1 100644 --- a/multipart/src/server/field.rs +++ b/multipart/src/server/field.rs @@ -21,18 +21,6 @@ use std::path::{Path, PathBuf}; use std::{str, fmt}; use std::error::Error; -macro_rules! try_io( - ($try:expr) => ( - { - use std::io::{Error, ErrorKind}; - match $try { - Ok(val) => val, - Err(e) => return Err(Error::new(ErrorKind::InvalidData, e)), - } - } - ) -); - const EMPTY_STR_HEADER: StrHeader<'static> = StrHeader { name: "", val: "", From 5765d0de017bb8a6c9afc45b507cf95383318263 Mon Sep 17 00:00:00 2001 From: Corentin Henry Date: Thu, 16 Mar 2017 11:35:52 -0700 Subject: [PATCH 288/453] prefer `try!` over `?` for error handling --- multipart/src/server/field.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/multipart/src/server/field.rs b/multipart/src/server/field.rs index e79d656e1..7cf0d5bfc 100644 --- a/multipart/src/server/field.rs +++ b/multipart/src/server/field.rs @@ -59,7 +59,7 @@ where R: BufRead, F: FnOnce(&[StrHeader]) -> Ret { Ok(Status::Complete((consume_, raw_headers))) => { consume = consume_; let mut headers = [EMPTY_STR_HEADER; HEADER_LEN]; - let headers = copy_headers(raw_headers, &mut headers)?; + let headers = try!(copy_headers(raw_headers, &mut headers)); debug!("Parsed headers: {:?}", headers); ret = closure(headers); break; @@ -96,11 +96,11 @@ pub struct FieldHeaders { impl FieldHeaders { /// Parse the field headers from the passed `BufRead`, consuming the relevant bytes. fn read_from(r: &mut R) -> Result { - with_headers(r, Self::parse)? + try!(with_headers(r, Self::parse)) } fn parse(headers: &[StrHeader]) -> Result { - let cont_disp = ContentDisp::parse(headers)?; + let cont_disp = try!(ContentDisp::parse(headers)); let cont_type = match parse_cont_type(headers) { Ok(cont_type) => Some(cont_type), // The Content-Type header is not mandatory so that's ok if it was not found. @@ -131,20 +131,20 @@ impl ContentDisp { } const CONT_DISP: &'static str = "Content-Disposition"; - let header = find_header(headers, CONT_DISP).ok_or(ParseHeaderError::Missing)?; + let header = try!(find_header(headers, CONT_DISP).ok_or(ParseHeaderError::Missing)); const NAME: &'static str = "name="; const FILENAME: &'static str = "filename="; let after_disp_type = { - let (disp_type, after_disp_type) = split_once(header.val, ';').ok_or(ParseHeaderError::Invalid)?; + let (disp_type, after_disp_type) = try!(split_once(header.val, ';').ok_or(ParseHeaderError::Invalid)); if disp_type.trim() != "form-data" { return Err(ParseHeaderError::Invalid); } after_disp_type }; - let (field_name, after_field_name) = get_str_after(NAME, ';', after_disp_type).ok_or(ParseHeaderError::Invalid)?; + let (field_name, after_field_name) = try!(get_str_after(NAME, ';', after_disp_type).ok_or(ParseHeaderError::Invalid)); let field_name = trim_quotes(field_name); let filename = get_str_after(FILENAME, ';', after_field_name) .map(|(filename, _)| trim_quotes(filename).to_owned()); @@ -156,7 +156,7 @@ impl ContentDisp { fn parse_cont_type(headers: &[StrHeader]) -> Result { const CONTENT_TYPE: &'static str = "Content-Type"; - let header = find_header(headers, CONTENT_TYPE).ok_or(ParseHeaderError::Missing)?; + let header = try!(find_header(headers, CONTENT_TYPE).ok_or(ParseHeaderError::Missing)); // Boundary parameter will be parsed into the `Mime` debug!("Found Content-Type: {:?}", header.val); From 66b882201def9e70c174f8d7d08419e1b3000ff4 Mon Sep 17 00:00:00 2001 From: Corentin Henry Date: Thu, 16 Mar 2017 20:43:33 -0700 Subject: [PATCH 289/453] not finding a header may or may not be an error parsers now return a `Result, ParserHeaderError>` instead of a `Result
, ParseHeaderError>` so that the caller can decide wether it is an error or not when a header is not found. --- multipart/src/server/field.rs | 44 +++++++++++++++-------------------- 1 file changed, 19 insertions(+), 25 deletions(-) diff --git a/multipart/src/server/field.rs b/multipart/src/server/field.rs index 7cf0d5bfc..d71598b93 100644 --- a/multipart/src/server/field.rs +++ b/multipart/src/server/field.rs @@ -100,18 +100,9 @@ impl FieldHeaders { } fn parse(headers: &[StrHeader]) -> Result { - let cont_disp = try!(ContentDisp::parse(headers)); - let cont_type = match parse_cont_type(headers) { - Ok(cont_type) => Some(cont_type), - // The Content-Type header is not mandatory so that's ok if it was not found. - Err(ParseHeaderError::Missing) => None, - // Any other error means the header was found but we failed to parse it. This is a read - // error. - Err(e) => return Err(e), - }; Ok(FieldHeaders { - cont_disp: cont_disp, - cont_type: cont_type, + cont_disp: try!(try!(ContentDisp::parse(headers)).ok_or(ParseHeaderError::NotFound)), + cont_type: try!(parse_cont_type(headers)), }) } } @@ -125,13 +116,13 @@ pub struct ContentDisp { } impl ContentDisp { - fn parse(headers: &[StrHeader]) -> Result { - if headers.is_empty() { - return Err(ParseHeaderError::Missing); - } - + fn parse(headers: &[StrHeader]) -> Result, ParseHeaderError> { const CONT_DISP: &'static str = "Content-Disposition"; - let header = try!(find_header(headers, CONT_DISP).ok_or(ParseHeaderError::Missing)); + let header = if let Some(header) = find_header(headers, CONT_DISP) { + header + } else { + return Ok(None); + }; const NAME: &'static str = "name="; const FILENAME: &'static str = "filename="; @@ -149,18 +140,21 @@ impl ContentDisp { let filename = get_str_after(FILENAME, ';', after_field_name) .map(|(filename, _)| trim_quotes(filename).to_owned()); - Ok(ContentDisp { field_name: field_name.to_owned(), filename: filename }) + Ok(Some(ContentDisp { field_name: field_name.to_owned(), filename: filename })) } } -fn parse_cont_type(headers: &[StrHeader]) -> Result { +fn parse_cont_type(headers: &[StrHeader]) -> Result, ParseHeaderError> { const CONTENT_TYPE: &'static str = "Content-Type"; - - let header = try!(find_header(headers, CONTENT_TYPE).ok_or(ParseHeaderError::Missing)); + let header = if let Some(header) = find_header(headers, CONTENT_TYPE) { + header + } else { + return Ok(None) + }; // Boundary parameter will be parsed into the `Mime` debug!("Found Content-Type: {:?}", header.val); - Ok(read_content_type(header.val.trim())) + Ok(Some(read_content_type(header.val.trim()))) } /// A field in a multipart request. May be either text or a binary stream (file). @@ -682,7 +676,7 @@ impl ReadEntryResult { #[derive(Debug)] pub enum ParseHeaderError { /// The header was not found - Missing, + NotFound, /// The header was found but could not be parsed Invalid, /// IO error @@ -693,7 +687,7 @@ pub enum ParseHeaderError { impl fmt::Display for ParseHeaderError { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match *self { - ParseHeaderError::Missing => write!(f, "header not found (ParseHeaderError::Missing)"), + ParseHeaderError::NotFound => write!(f, "header not found (ParseHeaderError::NotFound)"), ParseHeaderError::Invalid => write!(f, "invalid header (ParseHeaderError::Invalid)"), ParseHeaderError::Io(_) => write!(f, "could not read header (ParseHeaderError::Io)"), ParseHeaderError::Other(ref reason) => write!(f, "unknown parsing error (ParseHeaderError::Other(\"{}\"))", reason), @@ -704,7 +698,7 @@ impl fmt::Display for ParseHeaderError { impl Error for ParseHeaderError { fn description(&self) -> &str { match *self { - ParseHeaderError::Missing => "header not found", + ParseHeaderError::NotFound => "header not found", ParseHeaderError::Invalid => "the header is not formatted correctly", ParseHeaderError::Io(_) => "failed to read the header", ParseHeaderError::Other(_) => "unknown parsing error", From b1d3652aef271aabd0efeb663d3a9c150c3c8cf7 Mon Sep 17 00:00:00 2001 From: Corentin Henry Date: Thu, 16 Mar 2017 20:56:49 -0700 Subject: [PATCH 290/453] specify which header is not found in error Turn `ParseHeaderError::NotFound` into `ParseHeaderError::NotFound(HeaderType)`, so that the caller can know which header is missing. In practice, we only use the HeaderType::ContentDisposition so it may be a little bit overkill. --- multipart/src/server/field.rs | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/multipart/src/server/field.rs b/multipart/src/server/field.rs index d71598b93..a47786d23 100644 --- a/multipart/src/server/field.rs +++ b/multipart/src/server/field.rs @@ -100,8 +100,11 @@ impl FieldHeaders { } fn parse(headers: &[StrHeader]) -> Result { + let cont_disp = try!( + try!(ContentDisp::parse(headers)) + .ok_or(ParseHeaderError::NotFound(HeaderType::ContentDisposition))); Ok(FieldHeaders { - cont_disp: try!(try!(ContentDisp::parse(headers)).ok_or(ParseHeaderError::NotFound)), + cont_disp: cont_disp, cont_type: try!(parse_cont_type(headers)), }) } @@ -673,10 +676,17 @@ impl ReadEntryResult { } } + +#[derive(Debug)] +pub enum HeaderType { + ContentDisposition, + // ContentType, +} + #[derive(Debug)] pub enum ParseHeaderError { /// The header was not found - NotFound, + NotFound(HeaderType), /// The header was found but could not be parsed Invalid, /// IO error @@ -687,7 +697,7 @@ pub enum ParseHeaderError { impl fmt::Display for ParseHeaderError { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match *self { - ParseHeaderError::NotFound => write!(f, "header not found (ParseHeaderError::NotFound)"), + ParseHeaderError::NotFound(ref header_type) => write!(f, "header not found (ParseHeaderError::NotFound({:?}))", header_type), ParseHeaderError::Invalid => write!(f, "invalid header (ParseHeaderError::Invalid)"), ParseHeaderError::Io(_) => write!(f, "could not read header (ParseHeaderError::Io)"), ParseHeaderError::Other(ref reason) => write!(f, "unknown parsing error (ParseHeaderError::Other(\"{}\"))", reason), @@ -698,7 +708,7 @@ impl fmt::Display for ParseHeaderError { impl Error for ParseHeaderError { fn description(&self) -> &str { match *self { - ParseHeaderError::NotFound => "header not found", + ParseHeaderError::NotFound(_) => "header not found", ParseHeaderError::Invalid => "the header is not formatted correctly", ParseHeaderError::Io(_) => "failed to read the header", ParseHeaderError::Other(_) => "unknown parsing error", From 26ef5177ed274efa7178ff14eea6f2cb193a2c3a Mon Sep 17 00:00:00 2001 From: Corentin Henry Date: Thu, 16 Mar 2017 22:31:01 -0700 Subject: [PATCH 291/453] make ParseHeaderError private --- multipart/src/server/field.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/multipart/src/server/field.rs b/multipart/src/server/field.rs index a47786d23..bf96e391c 100644 --- a/multipart/src/server/field.rs +++ b/multipart/src/server/field.rs @@ -519,7 +519,7 @@ pub trait ReadEntry: PrivReadEntry + Sized { return End(self); } - let field_headers = try_read_entry!(self; self.read_headers().map_err(|e| io::Error::new(io::ErrorKind::Other, e))); + let field_headers = try_read_entry!(self; self.read_headers()); let data = match field_headers.cont_type { Some(cont_type) => { @@ -579,8 +579,9 @@ pub trait PrivReadEntry { /// Returns `true` if the last boundary was read, `false` otherwise. fn consume_boundary(&mut self) -> io::Result; - fn read_headers(&mut self) -> Result { + fn read_headers(&mut self) -> Result { FieldHeaders::read_from(&mut self.source()) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e)) } fn read_to_string(&mut self) -> io::Result { @@ -678,13 +679,13 @@ impl ReadEntryResult { #[derive(Debug)] -pub enum HeaderType { +enum HeaderType { ContentDisposition, // ContentType, } #[derive(Debug)] -pub enum ParseHeaderError { +enum ParseHeaderError { /// The header was not found NotFound(HeaderType), /// The header was found but could not be parsed From a544d784abd5ca6c803347c00a6e549e1f5acb4f Mon Sep 17 00:00:00 2001 From: Corentin Henry Date: Thu, 16 Mar 2017 23:05:37 -0700 Subject: [PATCH 292/453] make `ParseHeaderError::Invalid` more specific --- multipart/src/server/field.rs | 43 +++++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 15 deletions(-) diff --git a/multipart/src/server/field.rs b/multipart/src/server/field.rs index bf96e391c..2bfd09374 100644 --- a/multipart/src/server/field.rs +++ b/multipart/src/server/field.rs @@ -51,7 +51,7 @@ where R: BufRead, F: FnOnce(&[StrHeader]) -> Ret { let buf = try!(r.fill_buf()); if attempts == MAX_ATTEMPTS { - return Err(ParseHeaderError::Other("Unknown error while parsing headers".to_string())); + return Err(ParseHeaderError::Other("Could not read field headers".to_string())); } // FIXME: https://github.com/seanmonstar/httparse/issues/34 @@ -130,18 +130,31 @@ impl ContentDisp { const NAME: &'static str = "name="; const FILENAME: &'static str = "filename="; - let after_disp_type = { - let (disp_type, after_disp_type) = try!(split_once(header.val, ';').ok_or(ParseHeaderError::Invalid)); - if disp_type.trim() != "form-data" { - return Err(ParseHeaderError::Invalid); + let after_disp_type = match split_once(header.val, ';') { + Some((disp_type, after_disp_type)) => { + if disp_type.trim() != "form-data" { + let err = format!("Unexpected Content-Disposition value: {:?}", disp_type); + return Err(ParseHeaderError::Invalid(err)); + } + after_disp_type + }, + None => { + let err = format!("Expected additional data after Content-Disposition type, got {:?}", header.val); + return Err(ParseHeaderError::Invalid(err)); } - after_disp_type }; - let (field_name, after_field_name) = try!(get_str_after(NAME, ';', after_disp_type).ok_or(ParseHeaderError::Invalid)); - let field_name = trim_quotes(field_name); - let filename = get_str_after(FILENAME, ';', after_field_name) - .map(|(filename, _)| trim_quotes(filename).to_owned()); + let (field_name, filename) = match get_str_after(NAME, ';', after_disp_type) { + None => { + let err = format!("Expected field name and maybe filename, got {:?}", after_disp_type); + return Err(ParseHeaderError::Invalid(err)); + }, + Some((field_name, after_field_name)) => { + let field_name = trim_quotes(field_name); + let filename = get_str_after(FILENAME, ';', after_field_name).map(|(filename, _)| trim_quotes(filename).to_owned()); + (field_name, filename) + }, + }; Ok(Some(ContentDisp { field_name: field_name.to_owned(), filename: filename })) } @@ -689,7 +702,7 @@ enum ParseHeaderError { /// The header was not found NotFound(HeaderType), /// The header was found but could not be parsed - Invalid, + Invalid(String), /// IO error Io(io::Error), Other(String), @@ -699,7 +712,7 @@ impl fmt::Display for ParseHeaderError { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match *self { ParseHeaderError::NotFound(ref header_type) => write!(f, "header not found (ParseHeaderError::NotFound({:?}))", header_type), - ParseHeaderError::Invalid => write!(f, "invalid header (ParseHeaderError::Invalid)"), + ParseHeaderError::Invalid(ref msg) => write!(f, "invalid header (ParseHeaderError::Invalid({}))", msg), ParseHeaderError::Io(_) => write!(f, "could not read header (ParseHeaderError::Io)"), ParseHeaderError::Other(ref reason) => write!(f, "unknown parsing error (ParseHeaderError::Other(\"{}\"))", reason), } @@ -710,7 +723,7 @@ impl Error for ParseHeaderError { fn description(&self) -> &str { match *self { ParseHeaderError::NotFound(_) => "header not found", - ParseHeaderError::Invalid => "the header is not formatted correctly", + ParseHeaderError::Invalid(_) => "the header is not formatted correctly", ParseHeaderError::Io(_) => "failed to read the header", ParseHeaderError::Other(_) => "unknown parsing error", } @@ -731,7 +744,7 @@ impl From for ParseHeaderError { } impl From for ParseHeaderError { - fn from(_err: httparse::Error) -> ParseHeaderError { - ParseHeaderError::Invalid + fn from(err: httparse::Error) -> ParseHeaderError { + ParseHeaderError::Invalid(format!("{}", err)) } } From f86b6063ebb3798a87427fb54e873e7db59409a3 Mon Sep 17 00:00:00 2001 From: Corentin Henry Date: Thu, 16 Mar 2017 23:09:54 -0700 Subject: [PATCH 293/453] clippy: always run with nightly --- multipart/Cargo.toml | 2 +- multipart/src/client/lazy.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 4d6b24fb0..76ba8a121 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -47,7 +47,7 @@ default = ["all"] server = ["buf_redux", "httparse", "safemem", "twoway"] mock = [] nickel_ = ["nickel", "hyper"] -nightly = [] +nightly = ["clippy"] # Use this to enable SSE4.2 instructions in boundary finding # TODO: Benchmark this sse4 = ["nightly", "twoway/pcmp"] diff --git a/multipart/src/client/lazy.rs b/multipart/src/client/lazy.rs index 8988aa194..86012d0d1 100644 --- a/multipart/src/client/lazy.rs +++ b/multipart/src/client/lazy.rs @@ -507,7 +507,7 @@ mod hyper { impl<'d> super::PreparedFields<'d> { /// #### Feature: `hyper` /// Convert `self` to `hyper::client::Body`. - #[cfg_attr(feature="clippy", allow(wrong_self_convention))] + #[cfg_attr(feature="clippy", warn(wrong_self_convention))] pub fn to_body<'b>(&'b mut self) -> Body<'b> where 'd: 'b { if let Some(content_len) = self.content_len { Body::SizedBody(self, content_len) From cb77ba0f6c2365d1e61decd5786ceb3a985ddfb0 Mon Sep 17 00:00:00 2001 From: Corentin Henry Date: Fri, 17 Mar 2017 10:51:42 -0700 Subject: [PATCH 294/453] introduce ParseHeaderError::MissingContentDisposition --- multipart/src/server/field.rs | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/multipart/src/server/field.rs b/multipart/src/server/field.rs index 2bfd09374..f97187891 100644 --- a/multipart/src/server/field.rs +++ b/multipart/src/server/field.rs @@ -101,8 +101,7 @@ impl FieldHeaders { fn parse(headers: &[StrHeader]) -> Result { let cont_disp = try!( - try!(ContentDisp::parse(headers)) - .ok_or(ParseHeaderError::NotFound(HeaderType::ContentDisposition))); + try!(ContentDisp::parse(headers)).ok_or(ParseHeaderError::MissingContentDisposition)); Ok(FieldHeaders { cont_disp: cont_disp, cont_type: try!(parse_cont_type(headers)), @@ -691,16 +690,10 @@ impl ReadEntryResult { } -#[derive(Debug)] -enum HeaderType { - ContentDisposition, - // ContentType, -} - #[derive(Debug)] enum ParseHeaderError { - /// The header was not found - NotFound(HeaderType), + /// The `Content-Disposition` header was not found + MissingContentDisposition, /// The header was found but could not be parsed Invalid(String), /// IO error @@ -711,7 +704,7 @@ enum ParseHeaderError { impl fmt::Display for ParseHeaderError { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match *self { - ParseHeaderError::NotFound(ref header_type) => write!(f, "header not found (ParseHeaderError::NotFound({:?}))", header_type), + ParseHeaderError::MissingContentDisposition => write!(f, "\"Content-Disposition\" header not found (ParseHeaderError::MissingContentDisposition)"), ParseHeaderError::Invalid(ref msg) => write!(f, "invalid header (ParseHeaderError::Invalid({}))", msg), ParseHeaderError::Io(_) => write!(f, "could not read header (ParseHeaderError::Io)"), ParseHeaderError::Other(ref reason) => write!(f, "unknown parsing error (ParseHeaderError::Other(\"{}\"))", reason), @@ -722,7 +715,7 @@ impl fmt::Display for ParseHeaderError { impl Error for ParseHeaderError { fn description(&self) -> &str { match *self { - ParseHeaderError::NotFound(_) => "header not found", + ParseHeaderError::MissingContentDisposition => "\"Content-Disposition\" header not found", ParseHeaderError::Invalid(_) => "the header is not formatted correctly", ParseHeaderError::Io(_) => "failed to read the header", ParseHeaderError::Other(_) => "unknown parsing error", From 92ab0c2fb35c31816d5cff315651f9f22e6429ea Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Fri, 17 Mar 2017 18:54:52 -0700 Subject: [PATCH 295/453] Deprecate in-crate Nickel integration, move it to a separate crate in the same repo --- multipart/.gitignore | 6 +- multipart/README.md | 3 +- multipart/nickel/.gitignore | 3 + multipart/nickel/Cargo.toml | 13 +++ multipart/{ => nickel}/examples/nickel.rs | 12 ++- multipart/nickel/src/lib.rs | 121 ++++++++++++++++++++++ multipart/src/lib.rs | 5 + multipart/src/server/nickel.rs | 5 +- 8 files changed, 157 insertions(+), 11 deletions(-) create mode 100644 multipart/nickel/.gitignore create mode 100644 multipart/nickel/Cargo.toml rename multipart/{ => nickel}/examples/nickel.rs (92%) create mode 100644 multipart/nickel/src/lib.rs diff --git a/multipart/.gitignore b/multipart/.gitignore index fc01f5899..cb8fc54a6 100644 --- a/multipart/.gitignore +++ b/multipart/.gitignore @@ -12,8 +12,8 @@ *.exe # Generated by Cargo -/target/ - +target/ .idea/ +*.iml -dump.bin \ No newline at end of file +dump.bin diff --git a/multipart/README.md b/multipart/README.md index 92763cbcd..0bae61c1b 100644 --- a/multipart/README.md +++ b/multipart/README.md @@ -30,8 +30,7 @@ via the `tiny_http` feature. Provides server-side integration with `tiny_http::Request` via `multipart::server::Multipart`. -####[Nickel](http://nickel.rs/) (New in 0.6!) -via the `nickel_` feature +####[Nickel](http://nickel.rs/) (moved to `multipart-nickel` crate) Provides server-side integration with `&mut nickel::Request` via `multipart::server::Multipart`. diff --git a/multipart/nickel/.gitignore b/multipart/nickel/.gitignore new file mode 100644 index 000000000..c3ad9545f --- /dev/null +++ b/multipart/nickel/.gitignore @@ -0,0 +1,3 @@ +target/ +*.iml +Cargo.lock diff --git a/multipart/nickel/Cargo.toml b/multipart/nickel/Cargo.toml new file mode 100644 index 000000000..8567985b1 --- /dev/null +++ b/multipart/nickel/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "multipart-nickel" +version = "0.1.0" +authors = ["Austin Bonander "] + +description = "Support for `multipart/form-data` bodies in Nickel via the `multipart` crate." +documentation = "https://docs.rs/multipart-nickel" +repository = "https://github.com/abonander/multipart" + +[dependencies] +hyper = "0.9" +multipart = { version = "0.10", default-features = false, features = ["hyper", "server"] } +nickel = "0.9" diff --git a/multipart/examples/nickel.rs b/multipart/nickel/examples/nickel.rs similarity index 92% rename from multipart/examples/nickel.rs rename to multipart/nickel/examples/nickel.rs index 634c27602..f252c331a 100644 --- a/multipart/examples/nickel.rs +++ b/multipart/nickel/examples/nickel.rs @@ -1,14 +1,16 @@ +extern crate multipart_nickel; extern crate nickel; -extern crate multipart; use std::fs::File; use std::io::Read; -use multipart::server::{Entries, Multipart, SaveResult}; use nickel::{HttpRouter, MiddlewareResult, Nickel, Request, Response}; +use multipart_nickel::MultipartBody; +use multipart_nickel::multipart_server::{Entries, SaveResult}; + fn handle_multipart<'mw>(req: &mut Request, mut res: Response<'mw>) -> MiddlewareResult<'mw> { - match Multipart::from_request(req) { - Ok(mut multipart) => { + match req.multipart_body() { + Some(mut multipart) => { match multipart.save().temp() { SaveResult::Full(entries) => process_entries(res, entries), @@ -24,7 +26,7 @@ fn handle_multipart<'mw>(req: &mut Request, mut res: Response<'mw>) -> Middlewar }, } } - Err(_) => { + None => { res.set(nickel::status::StatusCode::BadRequest); return res.send("Request seems not was a multipart request") } diff --git a/multipart/nickel/src/lib.rs b/multipart/nickel/src/lib.rs new file mode 100644 index 000000000..def52d9e1 --- /dev/null +++ b/multipart/nickel/src/lib.rs @@ -0,0 +1,121 @@ +/// Support for `multipart/form-data` bodies in [Nickel](https://nickel.rs) via the +/// [`multipart`](https://crates.io/crates/multipart) crate. +/// +/// ### Why an external crate? +/// Three major reasons led to the decision to move Nickel integration to an external crate. +/// +/// 1: The part of Nickel's public API that matters to `multipart` (getting headers and +/// reading the request) uses Hyper's request type; this means that Nickel's integration +/// must necessarily be coupled to Hyper's integration. +/// +/// 2: Cargo does not allow specifying two different versions of the same crate in the +/// same manifest, probably for some good reasons but this crate's author has not looked into it. +/// +/// 3: Nickel's development moves incredibly slowly; when a new version of Hyper releases, there is +/// always a significant lag before Nickel upgrades, +/// [sometimes several months](https://github.com/nickel-org/nickel.rs/issues/367)--in this case, +/// Hyper was upgraded (in May) two months before the issue was opened (July), but a new version of +/// Nickel was not published until four months later (September). +/// +/// This causes problems for `multipart` because `multipart` cannot upgrade Hyper until Nickel does, +/// but its Hyper users often want to upgrade their Hyper as soon as possible. +/// +/// In order to provide up-to-date integration for Hyper, it was necessary to move Nickel +/// integration to an external crate so it can be pinned at the version of Hyper that Nickel +/// supports. This allows `multipart` to upgrade Hyper arbitrarily and still keep everyone happy. +/// +/// ### Porting from `multipart`'s Integration +/// +/// Whereas `multipart` only provided one way to wrap a Nickel request, this crate provides two: +/// +/// * To continue using `Multipart::from_request()`, wrap the request in +/// [`Maybe`](struct.Maybe.html): +/// +/// ```ignore +/// // Where `req` is `&mut nickel::Request` +/// - Multipart::from_request(req) +/// + use multipart_nickel::Maybe; +/// + Multipart::from_request(Maybe(req)) +/// ``` +/// +/// * Import `multipart_nickel::MultipartBody` and call `.multipart_body()`, which returns +/// `Option` (which better matches the conventions of `nickel::FormBody` and `nickel::JsonBody`): +/// +/// ```rust,ignore +/// use multipart_nickel::MultipartBody; +/// +/// // Where `req` is `&mut nickel::Request` +/// match req.multipart_body() { +/// Some(multipart) => // handle multipart body +/// None => // handle regular body +/// } +/// ``` +extern crate hyper; +extern crate multipart; +extern crate nickel; + +use nickel::Request as NickelRequest; + +use hyper::server::Request as HyperRequest; + +pub use multipart::server as multipart_server; + +use multipart_server::{HttpRequest, Multipart}; + +/// A wrapper for `&mut nickel::Request` which implements `multipart::server::HttpRequest`. +/// +/// Necessary because this crate cannot directly provide an impl of `HttpRequest` for +/// `&mut NickelRequest`. +pub struct Maybe<'r, 'mw: 'r, 'server: 'mw, D: 'mw>(pub &'r mut NickelRequest<'mw, 'server, D>); + +impl<'r, 'mw: 'r, 'server: 'mw, D: 'mw> HttpRequest for Maybe<'r, 'mw, 'server, D> { + type Body = &'r mut HyperRequest<'mw, 'server>; + + fn multipart_boundary(&self) -> Option<&str> { + self.0.origin.multipart_boundary() + } + + fn body(self) -> Self::Body { + &mut self.0.origin + } +} + +/// Extension trait for getting the `multipart/form-data` body from `nickel::Request`. +/// +/// Implemented for `nickel::Request`. +pub trait MultipartBody<'mw, 'server> { + /// Get a multipart reader for the request body, if the request is of the right type. + fn multipart_body(&mut self) -> Option>>; +} + +impl<'mw, 'server, D: 'mw> MultipartBody<'mw, 'server> for NickelRequest<'mw, 'server, D> { + fn multipart_body(&mut self) -> Option>> { + Multipart::from_request(Maybe(self)).ok() + } +} + + +impl<'r, 'mw: 'r, 'server: 'mw, D: 'mw> AsRef<&'r mut NickelRequest<'mw, 'server, D>> for Maybe<'r, 'mw, 'server, D> { + fn as_ref(&self) -> &&'r mut NickelRequest<'mw, 'server, D> { + &self.0 + } +} + +impl<'r, 'mw: 'r, 'server: 'mw, D: 'mw> AsMut<&'r mut NickelRequest<'mw, 'server, D>> for Maybe<'r, 'mw, 'server, D> { + fn as_mut(&mut self) -> &mut &'r mut NickelRequest<'mw, 'server, D> { + &mut self.0 + } +} + +impl<'r, 'mw: 'r, 'server: 'mw, D: 'mw> Into<&'r mut NickelRequest<'mw, 'server, D>> for Maybe<'r, 'mw, 'server, D> { + fn into(self) -> &'r mut NickelRequest<'mw, 'server, D> { + self.0 + } +} + +impl<'r, 'mw: 'r, 'server: 'mw, D: 'mw> From<&'r mut NickelRequest<'mw, 'server, D>> for Maybe<'r, 'mw, 'server, D> { + fn from(req: &'r mut NickelRequest<'mw, 'server, D>) -> Self { + Maybe(req) + } +} + diff --git a/multipart/src/lib.rs b/multipart/src/lib.rs index e5b94d290..68d6a0b7d 100644 --- a/multipart/src/lib.rs +++ b/multipart/src/lib.rs @@ -28,6 +28,11 @@ //! * `nickel_`: Integration with the [Nickel](http://nickel.rs) web application framework. //! See the [`server::nickel`](server/nickel/index.html) module for more information. Enables the `hyper` //! feature. +//! +//! **Note**: in-crate integration for Nickel is deprecated and will be removed in 0.11.0; +//! integration will be provided in the +//! [`multipart-nickel`](https://crates.io/crates/multipart-nickel) +//! crate for the foreseeable future. #![cfg_attr(feature="clippy", feature(plugin))] #![cfg_attr(feature="clippy", plugin(clippy))] #![cfg_attr(feature="clippy", deny(clippy))] diff --git a/multipart/src/server/nickel.rs b/multipart/src/server/nickel.rs index 5ac4da1d6..9d19488d4 100644 --- a/multipart/src/server/nickel.rs +++ b/multipart/src/server/nickel.rs @@ -2,13 +2,16 @@ //! (optional, enables `hyper` feature). //! //! Not shown here: [`impl HttpRequest for &mut nickel::Request`](../trait.HttpRequest.html#implementors). - +#![deprecated(since = "0.10.2", note = "Nickel integration has moved to the `multipart-nickel` + crate; in-crate integration will be removed in 0.11.0")] use nickel::Request as NickelRequest; use hyper::server::Request as HyperRequest; use super::HttpRequest; +#[deprecated(since = "0.10.2", note = "Nickel integration has moved to the `multipart-nickel` + crate; in-crate integration will be removed in 0.11.0")] impl<'r, 'mw, 'server, D: 'mw> HttpRequest for &'r mut NickelRequest<'mw, 'server, D> { type Body = &'r mut HyperRequest<'mw, 'server>; From 33f9bce2cf2a27c1d88e7c0bd79182c1b17d9c36 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Fri, 17 Mar 2017 20:36:06 -0700 Subject: [PATCH 296/453] Add `RUST_BACKTRACE=1` to Travis environment; test `multipart-nickel` --- multipart/.travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/multipart/.travis.yml b/multipart/.travis.yml index 7093210ea..e441177a9 100644 --- a/multipart/.travis.yml +++ b/multipart/.travis.yml @@ -14,7 +14,9 @@ os: env: # Get some logging in tests without being too verbose - RUST_LOG=multipart=info + - RUST_BACKTRACE=1 script: - cargo build -v --features all - cargo test -v --features all - cargo doc -v --no-deps --features all + - (cd nickel && cargo test -v) From 24b9e0868db23c12fba2ed055aecb2b6b7aba93a Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Fri, 17 Mar 2017 20:41:55 -0700 Subject: [PATCH 297/453] Fix docs in `multipart-nickel` --- multipart/nickel/src/lib.rs | 108 +++++++++++++++++++----------------- 1 file changed, 56 insertions(+), 52 deletions(-) diff --git a/multipart/nickel/src/lib.rs b/multipart/nickel/src/lib.rs index def52d9e1..20e048769 100644 --- a/multipart/nickel/src/lib.rs +++ b/multipart/nickel/src/lib.rs @@ -1,55 +1,58 @@ -/// Support for `multipart/form-data` bodies in [Nickel](https://nickel.rs) via the -/// [`multipart`](https://crates.io/crates/multipart) crate. -/// -/// ### Why an external crate? -/// Three major reasons led to the decision to move Nickel integration to an external crate. -/// -/// 1: The part of Nickel's public API that matters to `multipart` (getting headers and -/// reading the request) uses Hyper's request type; this means that Nickel's integration -/// must necessarily be coupled to Hyper's integration. -/// -/// 2: Cargo does not allow specifying two different versions of the same crate in the -/// same manifest, probably for some good reasons but this crate's author has not looked into it. -/// -/// 3: Nickel's development moves incredibly slowly; when a new version of Hyper releases, there is -/// always a significant lag before Nickel upgrades, -/// [sometimes several months](https://github.com/nickel-org/nickel.rs/issues/367)--in this case, -/// Hyper was upgraded (in May) two months before the issue was opened (July), but a new version of -/// Nickel was not published until four months later (September). -/// -/// This causes problems for `multipart` because `multipart` cannot upgrade Hyper until Nickel does, -/// but its Hyper users often want to upgrade their Hyper as soon as possible. -/// -/// In order to provide up-to-date integration for Hyper, it was necessary to move Nickel -/// integration to an external crate so it can be pinned at the version of Hyper that Nickel -/// supports. This allows `multipart` to upgrade Hyper arbitrarily and still keep everyone happy. -/// -/// ### Porting from `multipart`'s Integration -/// -/// Whereas `multipart` only provided one way to wrap a Nickel request, this crate provides two: -/// -/// * To continue using `Multipart::from_request()`, wrap the request in -/// [`Maybe`](struct.Maybe.html): -/// -/// ```ignore -/// // Where `req` is `&mut nickel::Request` -/// - Multipart::from_request(req) -/// + use multipart_nickel::Maybe; -/// + Multipart::from_request(Maybe(req)) -/// ``` -/// -/// * Import `multipart_nickel::MultipartBody` and call `.multipart_body()`, which returns -/// `Option` (which better matches the conventions of `nickel::FormBody` and `nickel::JsonBody`): -/// -/// ```rust,ignore -/// use multipart_nickel::MultipartBody; -/// -/// // Where `req` is `&mut nickel::Request` -/// match req.multipart_body() { -/// Some(multipart) => // handle multipart body -/// None => // handle regular body -/// } -/// ``` +//! Support for `multipart/form-data` bodies in [Nickel](https://nickel.rs) via the +//! [`multipart`](https://crates.io/crates/multipart) crate. +//! +//! ### Why an external crate? +//! Three major reasons led to the decision to move Nickel integration to an external crate. +//! +//! 1: The part of Nickel's public API that matters to `multipart` (getting headers and +//! reading the request) uses Hyper's request type; this means that Nickel's integration +//! must necessarily be coupled to Hyper's integration. +//! +//! 2: Cargo does not allow specifying two different versions of the same crate in the +//! same manifest, probably for some good reasons but this crate's author has not looked into it. +//! +//! 3: Nickel's development moves incredibly slowly; when a new version of Hyper releases, there is +//! always a significant lag before Nickel upgrades, +//! [sometimes several months](https://github.com/nickel-org/nickel.rs/issues/367)--in this case, +//! Hyper was upgraded (in May) two months before the issue was opened (July), but a new version of +//! Nickel was not published until four months later (September). +//! +//! This causes problems for `multipart` because it cannot upgrade Hyper until Nickel does, +//! but its Hyper users often want to upgrade their Hyper as soon as possible. +//! +//! In order to provide up-to-date integration for Hyper, it was necessary to move Nickel +//! integration to an external crate so it can be pinned at the version of Hyper that Nickel +//! supports. This allows `multipart` to upgrade Hyper arbitrarily and still keep everyone happy. +//! +//! ### Porting from `multipart`'s Integration +//! +//! Whereas `multipart` only provided one way to wrap a Nickel request, this crate provides two: +//! +//! * To continue using `Multipart::from_request()`, wrap the request in +//! [`Maybe`](struct.Maybe.html): +//! +//! ```ignore +//! // Where `req` is `&mut nickel::Request` +//! - Multipart::from_request(req) +//! + use multipart_nickel::Maybe; +//! + Multipart::from_request(Maybe(req)) +//! ``` +//! +//! * Import `multipart_nickel::MultipartBody` and call `.multipart_body()`, which returns +//! `Option` (which better matches the conventions of `nickel::FormBody` and `nickel::JsonBody`): +//! +//! ```rust,ignore +//! use multipart_nickel::MultipartBody; +//! +//! // Where `req` is `&mut nickel::Request` +//! match req.multipart_body() { +//! Some(multipart) => // handle multipart body +//! None => // handle regular body +//! } +//! ``` +//! +//! This crate also reexports the `server` module from the version of `multipart` +//! that it uses, so that you don't have to import `multipart` separately. extern crate hyper; extern crate multipart; extern crate nickel; @@ -58,6 +61,7 @@ use nickel::Request as NickelRequest; use hyper::server::Request as HyperRequest; +/// The `server` module from the version of `multipart` that this crate uses. pub use multipart::server as multipart_server; use multipart_server::{HttpRequest, Multipart}; From d00c4312991f7c0a304d733f8d0585c3e6c2e081 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Fri, 17 Mar 2017 20:53:57 -0700 Subject: [PATCH 298/453] Fix Travis config --- multipart/.travis.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/multipart/.travis.yml b/multipart/.travis.yml index e441177a9..51619b37b 100644 --- a/multipart/.travis.yml +++ b/multipart/.travis.yml @@ -12,9 +12,7 @@ os: - linux - osx env: - # Get some logging in tests without being too verbose - - RUST_LOG=multipart=info - - RUST_BACKTRACE=1 + - RUST_LOG=multipart=info RUST_BACKTRACE=1 script: - cargo build -v --features all - cargo test -v --features all From ff55f370620ffc82ef2b927044e0b7afce10da74 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Fri, 17 Mar 2017 21:09:41 -0700 Subject: [PATCH 299/453] Disable nightly branch until Cargo fix hits the trains --- multipart/.travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/multipart/.travis.yml b/multipart/.travis.yml index 51619b37b..c2dde49b9 100644 --- a/multipart/.travis.yml +++ b/multipart/.travis.yml @@ -7,7 +7,8 @@ branches: rust: - stable - beta - - nightly + # FIXME: when https://github.com/rust-lang/cargo/issues/3844 is closed + # - nightly os: - linux - osx From 2fa3af09e9474d5e1429b63c997435a4769e85d3 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Fri, 17 Mar 2017 21:48:13 -0700 Subject: [PATCH 300/453] Also disable builds on OS X for the time being --- multipart/.travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/multipart/.travis.yml b/multipart/.travis.yml index c2dde49b9..5b57f3127 100644 --- a/multipart/.travis.yml +++ b/multipart/.travis.yml @@ -11,7 +11,8 @@ rust: # - nightly os: - linux - - osx + # openssl is broken on OS X again + # - osx env: - RUST_LOG=multipart=info RUST_BACKTRACE=1 script: From fa43ba0dd258827c01703eacc03258305a90c494 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Mon, 20 Mar 2017 16:21:22 -0700 Subject: [PATCH 301/453] Add deprecation note to `nickel` module and a warning log message in the impl --- multipart/src/server/nickel.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/multipart/src/server/nickel.rs b/multipart/src/server/nickel.rs index 9d19488d4..d946f5d08 100644 --- a/multipart/src/server/nickel.rs +++ b/multipart/src/server/nickel.rs @@ -2,6 +2,11 @@ //! (optional, enables `hyper` feature). //! //! Not shown here: [`impl HttpRequest for &mut nickel::Request`](../trait.HttpRequest.html#implementors). +//! +//! **Note**: in-crate integration for Nickel is deprecated and will be removed in 0.11.0; +//! integration will be provided in the +//! [`multipart-nickel`](https://crates.io/crates/multipart-nickel) +//! crate for the foreseeable future. #![deprecated(since = "0.10.2", note = "Nickel integration has moved to the `multipart-nickel` crate; in-crate integration will be removed in 0.11.0")] use nickel::Request as NickelRequest; @@ -20,6 +25,9 @@ impl<'r, 'mw, 'server, D: 'mw> HttpRequest for &'r mut NickelRequest<'mw, 'serve } fn body(self) -> Self::Body { + warn!("In-crate Nickel integration is deprecated and will be removed in 0.11.0; \ + please consider switching to the `multipart-nickel` crate to avoid breakage."); + &mut self.origin } } From 3bb35b46184afc32236884b19eb87fcfa9e19e92 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Mon, 20 Mar 2017 16:22:52 -0700 Subject: [PATCH 302/453] Lower deprecation log message to `info` --- multipart/src/server/nickel.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/src/server/nickel.rs b/multipart/src/server/nickel.rs index d946f5d08..16ebe22b3 100644 --- a/multipart/src/server/nickel.rs +++ b/multipart/src/server/nickel.rs @@ -25,7 +25,7 @@ impl<'r, 'mw, 'server, D: 'mw> HttpRequest for &'r mut NickelRequest<'mw, 'serve } fn body(self) -> Self::Body { - warn!("In-crate Nickel integration is deprecated and will be removed in 0.11.0; \ + info!("In-crate Nickel integration is deprecated and will be removed in 0.11.0; \ please consider switching to the `multipart-nickel` crate to avoid breakage."); &mut self.origin From b73b32b57d60f7c319a4e9732e2d25b1ec12163f Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Mon, 20 Mar 2017 16:40:40 -0700 Subject: [PATCH 303/453] Keep old `nickel` example and add deprecation note --- multipart/examples/nickel.rs | 87 ++++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 multipart/examples/nickel.rs diff --git a/multipart/examples/nickel.rs b/multipart/examples/nickel.rs new file mode 100644 index 000000000..576659f7e --- /dev/null +++ b/multipart/examples/nickel.rs @@ -0,0 +1,87 @@ +//! **Note**: in-crate integration for Nickel is deprecated and will be removed in 0.11.0; +//! integration will be provided in the +//! [`multipart-nickel`](https://crates.io/crates/multipart-nickel) +//! crate for the foreseeable future. +extern crate multipart; +extern crate nickel; + +use std::fs::File; +use std::io::Read; +use nickel::{HttpRouter, MiddlewareResult, Nickel, Request, Response}; + +use multipart::server::{Multipart, Entries, SaveResult}; + +fn handle_multipart<'mw>(req: &mut Request, mut res: Response<'mw>) -> MiddlewareResult<'mw> { + match Multipart::from_request(req) { + Ok(mut multipart) => { + match multipart.save().temp() { + SaveResult::Full(entries) => process_entries(res, entries), + + SaveResult::Partial(entries, e) => { + println!("Partial errors ... {:?}", e); + return process_entries(res, entries.keep_partial()); + }, + + SaveResult::Error(e) => { + println!("There are errors in multipart POSTing ... {:?}", e); + res.set(nickel::status::StatusCode::InternalServerError); + return res.send(format!("Server could not handle multipart POST! {:?}", e)); + }, + } + } + Err(_) => { + res.set(nickel::status::StatusCode::BadRequest); + return res.send("Request seems not was a multipart request") + } + } +} + +/// Processes saved entries from multipart request. +/// Returns an OK response or an error. +fn process_entries<'mw>(res: Response<'mw>, entries: Entries) -> MiddlewareResult<'mw> { + for (name, field) in entries.fields { + println!("Field {:?}: {:?}", name, field); + } + + for (name, files) in entries.files { + println!("Field {:?} has {} files:", name, files.len()); + + for saved_file in files { + match File::open(&saved_file.path) { + Ok(mut file) => { + let mut contents = String::new(); + if let Err(e) = file.read_to_string(&mut contents) { + println!("Could not read file {:?}. Error: {:?}", saved_file.filename, e); + return res.error(nickel::status::StatusCode::BadRequest, "The uploaded file was not readable") + } + + println!("File {:?} ({:?}):", saved_file.filename, saved_file.content_type); + println!("{}", contents); + file + } + Err(e) => { + println!("Could open file {:?}. Error: {:?}", saved_file.filename, e); + return res.error(nickel::status::StatusCode::BadRequest, "The uploaded file was not readable") + } + }; + } + } + + res.send("Ok") +} + +fn main() { + let mut srv = Nickel::new(); + + srv.post("/multipart_upload/", handle_multipart); + + // Start this example via: + // + // `cargo run --example nickel --features nickel` + // + // And - if you are in the root of this repository - do an example + // upload via: + // + // `curl -F file=@LICENSE 'http://localhost:6868/multipart_upload/'` + srv.listen("127.0.0.1:6868").expect("Failed to bind server"); +} From 3f9aea0711ed6cc203702211d79d2348be750323 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Mon, 20 Mar 2017 16:43:42 -0700 Subject: [PATCH 304/453] Clarify Nickel integration change in README --- multipart/README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/multipart/README.md b/multipart/README.md index 0bae61c1b..d8018fbec 100644 --- a/multipart/README.md +++ b/multipart/README.md @@ -30,7 +30,10 @@ via the `tiny_http` feature. Provides server-side integration with `tiny_http::Request` via `multipart::server::Multipart`. -####[Nickel](http://nickel.rs/) (moved to `multipart-nickel` crate) +####[Nickel](http://nickel.rs/) + +**Note**: Moved to `multipart-nickel` crate, see [nickel/examples/nickel.rs](nickel/examples/nickel.rs) +for updated integration example. Provides server-side integration with `&mut nickel::Request` via `multipart::server::Multipart`. From 114b97f832f3fa0537f2759166d4e2b62e581ae2 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Tue, 21 Mar 2017 15:33:21 -0700 Subject: [PATCH 305/453] multipart 0.10.2 --- multipart/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 76ba8a121..f9e29908d 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "multipart" -version = "0.10.1" +version = "0.10.2" authors = ["Austin Bonander "] From b56b5113bfd76c593c657dbe925a8bbc47af608c Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Tue, 21 Mar 2017 15:39:34 -0700 Subject: [PATCH 306/453] Add license key to nickel/Cargo.toml --- multipart/nickel/Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/multipart/nickel/Cargo.toml b/multipart/nickel/Cargo.toml index 8567985b1..cb5ed037d 100644 --- a/multipart/nickel/Cargo.toml +++ b/multipart/nickel/Cargo.toml @@ -6,6 +6,7 @@ authors = ["Austin Bonander "] description = "Support for `multipart/form-data` bodies in Nickel via the `multipart` crate." documentation = "https://docs.rs/multipart-nickel" repository = "https://github.com/abonander/multipart" +license = "MIT OR Apache-2.0" [dependencies] hyper = "0.9" From b7785010965c0e0afb0f7f99b5578ad98843955e Mon Sep 17 00:00:00 2001 From: Corentin Henry Date: Sat, 25 Mar 2017 17:02:36 -0700 Subject: [PATCH 307/453] fix build for rust >= 1.12.0 --- multipart/src/server/field.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/multipart/src/server/field.rs b/multipart/src/server/field.rs index f97187891..01dd8bb64 100644 --- a/multipart/src/server/field.rs +++ b/multipart/src/server/field.rs @@ -18,8 +18,7 @@ use mime::{TopLevel, Mime}; use std::io::{self, Read, BufRead, Write}; use std::ops::Deref; use std::path::{Path, PathBuf}; -use std::{str, fmt}; -use std::error::Error; +use std::{str, fmt, error}; const EMPTY_STR_HEADER: StrHeader<'static> = StrHeader { name: "", @@ -712,7 +711,7 @@ impl fmt::Display for ParseHeaderError { } } -impl Error for ParseHeaderError { +impl error::Error for ParseHeaderError { fn description(&self) -> &str { match *self { ParseHeaderError::MissingContentDisposition => "\"Content-Disposition\" header not found", @@ -722,7 +721,7 @@ impl Error for ParseHeaderError { } } - fn cause(&self) -> Option<&Error> { + fn cause(&self) -> Option<&error::Error> { match *self { ParseHeaderError::Io(ref e) => Some(e), _ => None, From 81abb406580df63d8c58a4eae252a5a2ab81db8f Mon Sep 17 00:00:00 2001 From: Corentin Henry Date: Sat, 25 Mar 2017 17:13:28 -0700 Subject: [PATCH 308/453] fix tests for rust < 1.15.0 --- multipart/src/local_test.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/multipart/src/local_test.rs b/multipart/src/local_test.rs index 5407619fd..1bbec435f 100644 --- a/multipart/src/local_test.rs +++ b/multipart/src/local_test.rs @@ -199,7 +199,7 @@ fn lazy_client_entry_server() { } mod extended { - use super::*; + use super::{test_client, test_server, test_server_entry_api, test_client_lazy, TestFields}; use std::time::Instant; @@ -384,4 +384,4 @@ fn rand_mime() -> Mime { mime!(Text/Plain), mime!(Image/Png), ]).unwrap().clone() -} \ No newline at end of file +} From 3aceb1bac4fb67a7cd6bdac43184bf7a1fcb3a9b Mon Sep 17 00:00:00 2001 From: Corentin Henry Date: Sat, 25 Mar 2017 17:03:33 -0700 Subject: [PATCH 309/453] replace try! by the question mark operator See https://github.com/japaric/untry --- multipart/examples/hyper_client.rs | 4 ++-- multipart/examples/hyper_server.rs | 6 +++--- multipart/examples/iron.rs | 4 ++-- multipart/examples/tiny_http.rs | 8 ++++---- multipart/src/client/lazy.rs | 4 ++-- multipart/src/client/mod.rs | 8 ++++---- multipart/src/client/sized.rs | 4 ++-- multipart/src/local_test.rs | 6 +++--- multipart/src/server/boundary.rs | 12 ++++++------ multipart/src/server/field.rs | 13 ++++++------- multipart/src/server/iron.rs | 8 +++----- 11 files changed, 37 insertions(+), 40 deletions(-) diff --git a/multipart/examples/hyper_client.rs b/multipart/examples/hyper_client.rs index 2c8766681..0684fecc8 100644 --- a/multipart/examples/hyper_client.rs +++ b/multipart/examples/hyper_client.rs @@ -28,8 +28,8 @@ fn main() { fn write_body(multi: &mut Multipart>) -> hyper::Result<()> { let mut binary = "Hello world from binary!".as_bytes(); - try!(multi.write_text("text", "Hello, world!")); - try!(multi.write_file("file", "lorem_ipsum.txt")); + multi.write_text("text", "Hello, world!")?; + multi.write_file("file", "lorem_ipsum.txt")?; // &[u8] impl Read multi.write_stream("binary", &mut binary, None, None) .and(Ok(())) diff --git a/multipart/examples/hyper_server.rs b/multipart/examples/hyper_server.rs index c100a92e5..e3869027e 100644 --- a/multipart/examples/hyper_server.rs +++ b/multipart/examples/hyper_server.rs @@ -50,7 +50,7 @@ fn process_entries<'a>(entries: Entries) -> io::Result<()> { println!("Field {:?} has {} files:", name, files.len()); for file in files { - try!(print_file(&file)); + print_file(&file)?; } } @@ -58,10 +58,10 @@ fn process_entries<'a>(entries: Entries) -> io::Result<()> { } fn print_file(saved_file: &SavedFile) -> io::Result<()> { - let mut file = try!(File::open(&saved_file.path)); + let mut file = File::open(&saved_file.path)?; let mut contents = String::new(); - try!(file.read_to_string(&mut contents)); + file.read_to_string(&mut contents)?; println!("File {:?} ({:?}):", saved_file.filename, saved_file.content_type); println!("{}", contents); diff --git a/multipart/examples/iron.rs b/multipart/examples/iron.rs index 5a972d464..4292a2d88 100644 --- a/multipart/examples/iron.rs +++ b/multipart/examples/iron.rs @@ -22,7 +22,7 @@ fn process_request(request: &mut Request) -> IronResult { match multipart.save().temp() { SaveResult::Full(entries) => process_entries(entries), SaveResult::Partial(entries, reason) => { - try!(process_entries(entries.keep_partial())); + process_entries(entries.keep_partial())?; Err(IronError::new(reason.unwrap_err(), status::InternalServerError)) } SaveResult::Error(error) => Err(IronError::new(error, status::InternalServerError)), @@ -45,7 +45,7 @@ fn process_entries(entries: Entries) -> IronResult { println!("Field {:?} has {} files:", name, files.len()); for file in files { - try!(print_file(&file)); + print_file(&file)?; } } diff --git a/multipart/examples/tiny_http.rs b/multipart/examples/tiny_http.rs index 0c7b4d4af..9571b8e82 100644 --- a/multipart/examples/tiny_http.rs +++ b/multipart/examples/tiny_http.rs @@ -38,7 +38,7 @@ fn process_request<'a, 'b>(request: &'a mut Request) -> io::Result process_entries(entries), SaveResult::Partial(entries, reason) => { - try!(process_entries(entries.keep_partial())); + process_entries(entries.keep_partial())?; // We don't set limits Err(reason.unwrap_err()) } @@ -60,7 +60,7 @@ fn process_entries<'a>(entries: Entries) -> io::Result> { println!("Field {:?} has {} files:", name, files.len()); for file in files { - try!(print_file(&file)); + print_file(&file)?; } } @@ -68,10 +68,10 @@ fn process_entries<'a>(entries: Entries) -> io::Result> { } fn print_file(saved_file: &SavedFile) -> io::Result<()> { - let mut file = try!(File::open(&saved_file.path)); + let mut file = File::open(&saved_file.path)?; let mut contents = String::new(); - try!(file.read_to_string(&mut contents)); + file.read_to_string(&mut contents)?; println!("File {:?} ({:?}):", saved_file.filename, saved_file.content_type); println!("{}", contents); diff --git a/multipart/src/client/lazy.rs b/multipart/src/client/lazy.rs index 86012d0d1..df53ec165 100644 --- a/multipart/src/client/lazy.rs +++ b/multipart/src/client/lazy.rs @@ -265,7 +265,7 @@ impl<'d> PreparedFields<'d> { let mut writer = MultipartWriter::new(Vec::new(), &*boundary); while fields.peek().is_some() { - if let Some(rem) = try!(write_fields(&mut writer, &mut fields, buffer_threshold)) { + if let Some(rem) = write_fields(&mut writer, &mut fields, buffer_threshold)? { let contiguous = mem::replace(writer.inner_mut(), Vec::new()); prep_fields.push(PreparedField::Partial(Cursor::new(contiguous), rem)); } @@ -317,7 +317,7 @@ impl<'d> Read for PreparedFields<'d> { let buf = &mut buf[total_read..]; let num_read = if let Some(ref mut field) = self.next_field { - try!(field.read(buf)) + field.read(buf)? } else { break; }; diff --git a/multipart/src/client/mod.rs b/multipart/src/client/mod.rs index d7f963cae..89cab11ad 100644 --- a/multipart/src/client/mod.rs +++ b/multipart/src/client/mod.rs @@ -51,7 +51,7 @@ impl Multipart<()> { /// ## Returns Error /// If `req.open_stream()` returns an error. pub fn from_request(req: R) -> Result, R::Error> { - let (boundary, stream) = try!(open_stream(req, None)); + let (boundary, stream) = open_stream(req, None)?; Ok(Multipart { writer: MultipartWriter::new(stream, boundary), @@ -206,7 +206,7 @@ impl<'a, W: Write> MultipartWriter<'a, W> { fn write_boundary(&mut self) -> io::Result<()> { if self.data_written { - try!(self.inner.write_all(b"\r\n")); + self.inner.write_all(b"\r\n")?; } write!(self.inner, "--{}\r\n", self.boundary) @@ -221,7 +221,7 @@ impl<'a, W: Write> MultipartWriter<'a, W> { fn write_file(&mut self, name: &str, path: &Path) -> io::Result<()> { let (content_type, filename) = mime_filename(path); - let mut file = try!(File::open(path)); + let mut file = File::open(path)?; self.write_stream(&mut file, name, filename, Some(content_type)) } @@ -258,7 +258,7 @@ impl<'a, W: Write> MultipartWriter<'a, W> { fn finish(mut self) -> io::Result { if self.data_written { // Write two hyphens after the last boundary occurrence. - try!(write!(self.inner, "\r\n--{}--", self.boundary)); + write!(self.inner, "\r\n--{}--", self.boundary)?; } Ok(self.inner) diff --git a/multipart/src/client/sized.rs b/multipart/src/client/sized.rs index ed2e7825b..e7caf6b3f 100644 --- a/multipart/src/client/sized.rs +++ b/multipart/src/client/sized.rs @@ -83,8 +83,8 @@ where ::Error: From { ).into()); } - let mut req = try!(self.inner.open_stream()); - try!(io::copy(&mut &self.buffer[..], &mut req)); + let mut req = self.inner.open_stream()?; + io::copy(&mut &self.buffer[..], &mut req)?; req.finish().into() } } diff --git a/multipart/src/local_test.rs b/multipart/src/local_test.rs index 1bbec435f..bd8ef261e 100644 --- a/multipart/src/local_test.rs +++ b/multipart/src/local_test.rs @@ -136,15 +136,15 @@ struct PrintHex(Vec); impl fmt::Debug for PrintHex { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - try!(write!(f, "[")); + write!(f, "[")?; let mut written = false; for byte in &self.0 { - try!(write!(f, "{:X}", byte)); + write!(f, "{:X}", byte)?; if written { - try!(write!(f, ", ")); + write!(f, ", ")?; } written = true; diff --git a/multipart/src/server/boundary.rs b/multipart/src/server/boundary.rs index 52ee2a4de..1d4ecfd3d 100644 --- a/multipart/src/server/boundary.rs +++ b/multipart/src/server/boundary.rs @@ -47,7 +47,7 @@ impl BoundaryReader where R: Read { // Make sure there's enough bytes in the buffer to positively identify the boundary. let min_len = self.search_idx + (self.boundary.len() * 2); - let buf = try!(fill_buf_min(&mut self.source, min_len)); + let buf = fill_buf_min(&mut self.source, min_len)?; if buf.is_empty() { debug!("fill_buf_min returned zero-sized buf"); @@ -118,7 +118,7 @@ impl BoundaryReader where R: Read { while !(self.boundary_read || self.at_end){ debug!("Boundary not found yet"); - let buf_len = try!(self.read_to_boundary()).len(); + let buf_len = self.read_to_boundary()?.len(); debug!("Discarding {} bytes", buf_len); @@ -147,10 +147,10 @@ impl BoundaryReader where R: Read { let mut bytes_after = [0, 0]; - let read = try!(self.source.read(&mut bytes_after)); + let read = self.source.read(&mut bytes_after)?; if read == 1 { - let _ = try!(self.source.read(&mut bytes_after[1..])); + let _ = self.source.read(&mut bytes_after[1..])?; } if bytes_after == *b"--" { @@ -179,7 +179,7 @@ impl Borrow for BoundaryReader { impl Read for BoundaryReader where R: Read { fn read(&mut self, out: &mut [u8]) -> io::Result { let read = { - let mut buf = try!(self.read_to_boundary()); + let mut buf = self.read_to_boundary()?; // This shouldn't ever be an error so unwrapping is fine. buf.read(out).unwrap() }; @@ -210,7 +210,7 @@ fn fill_buf_min(buf: &mut BufReader, min: usize) -> io::Result<&[u8] let mut attempts = 0; while buf.available() < min && attempts < MAX_ATTEMPTS { - if try!(buf.read_into_buf()) == 0 { break; }; + if buf.read_into_buf()? == 0 { break; }; attempts += 1; } diff --git a/multipart/src/server/field.rs b/multipart/src/server/field.rs index 01dd8bb64..d1f82048b 100644 --- a/multipart/src/server/field.rs +++ b/multipart/src/server/field.rs @@ -47,7 +47,7 @@ where R: BufRead, F: FnOnce(&[StrHeader]) -> Ret { loop { let mut raw_headers = [EMPTY_HEADER; HEADER_LEN]; - let buf = try!(r.fill_buf()); + let buf = r.fill_buf()?; if attempts == MAX_ATTEMPTS { return Err(ParseHeaderError::Other("Could not read field headers".to_string())); @@ -58,7 +58,7 @@ where R: BufRead, F: FnOnce(&[StrHeader]) -> Ret { Ok(Status::Complete((consume_, raw_headers))) => { consume = consume_; let mut headers = [EMPTY_STR_HEADER; HEADER_LEN]; - let headers = try!(copy_headers(raw_headers, &mut headers)); + let headers = copy_headers(raw_headers, &mut headers)?; debug!("Parsed headers: {:?}", headers); ret = closure(headers); break; @@ -78,7 +78,7 @@ where R: BufRead, F: FnOnce(&[StrHeader]) -> Ret { fn copy_headers<'h, 'b: 'h>(raw: &[Header<'b>], headers: &'h mut [StrHeader<'b>]) -> io::Result<&'h [StrHeader<'b>]> { for (raw, header) in raw.iter().zip(&mut *headers) { header.name = raw.name; - header.val = try!(io_str_utf8(raw.value)); + header.val = io_str_utf8(raw.value)?; } Ok(&headers[..raw.len()]) @@ -95,15 +95,14 @@ pub struct FieldHeaders { impl FieldHeaders { /// Parse the field headers from the passed `BufRead`, consuming the relevant bytes. fn read_from(r: &mut R) -> Result { - try!(with_headers(r, Self::parse)) + with_headers(r, Self::parse)? } fn parse(headers: &[StrHeader]) -> Result { - let cont_disp = try!( - try!(ContentDisp::parse(headers)).ok_or(ParseHeaderError::MissingContentDisposition)); + let cont_disp = ContentDisp::parse(headers)?.ok_or(ParseHeaderError::MissingContentDisposition)?; Ok(FieldHeaders { cont_disp: cont_disp, - cont_type: try!(parse_cont_type(headers)), + cont_type: parse_cont_type(headers)?, }) } } diff --git a/multipart/src/server/iron.rs b/multipart/src/server/iron.rs index c1195e3c5..ece7aec42 100644 --- a/multipart/src/server/iron.rs +++ b/multipart/src/server/iron.rs @@ -124,14 +124,12 @@ impl Intercept { Err(_) => return Ok(None), }; - let tempdir = try!( - self.temp_dir_path.as_ref() + let tempdir = self.temp_dir_path.as_ref() .map_or_else( || TempDir::new("multipart-iron"), |path| TempDir::new_in(path, "multipart-iron") ) - .map_err(|e| io_to_iron(e, "Error opening temporary directory for request.")) - ); + .map_err(|e| io_to_iron(e, "Error opening temporary directory for request."))?; match self.limit_behavior { LimitBehavior::ThrowError => self.read_request_strict(multipart, tempdir), @@ -200,7 +198,7 @@ impl Default for Intercept { impl BeforeMiddleware for Intercept { fn before(&self, req: &mut IronRequest) -> IronResult<()> { - try!(self.read_request(req)) + self.read_request(req)? .map(|entries| req.extensions.insert::(entries)); Ok(()) From f0c9c475e1b81a9ede55f176793dd25836de6a4c Mon Sep 17 00:00:00 2001 From: chpio Date: Wed, 29 Mar 2017 14:09:04 +0000 Subject: [PATCH 310/453] fix headings in README --- multipart/README.md | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/multipart/README.md b/multipart/README.md index d8018fbec..89e6017d5 100644 --- a/multipart/README.md +++ b/multipart/README.md @@ -1,17 +1,16 @@ -Multipart [![Build Status](https://travis-ci.org/abonander/multipart.svg?branch=master)](https://travis-ci.org/abonander/multipart) [![On Crates.io](https://img.shields.io/crates/v/multipart.svg)](https://crates.io/crates/multipart) -========= +# Multipart [![Build Status](https://travis-ci.org/abonander/multipart.svg?branch=master)](https://travis-ci.org/abonander/multipart) [![On Crates.io](https://img.shields.io/crates/v/multipart.svg)](https://crates.io/crates/multipart) Client- and server-side abstractions for HTTP file uploads (POST requests with `Content-Type: multipart/form-data`). Supports several different HTTP crates. -###[Documentation](http://docs.rs/multipart/) +### [Documentation](http://docs.rs/multipart/) -##Integrations +## Integrations Example files demonstrating how to use `multipart` with these crates are available under [`examples/`](examples). -####[Hyper](http://hyper.rs) +### [Hyper](http://hyper.rs) via the `hyper` feature (enabled by default). Client integration includes support for regular `hyper::client::Request` objects via `multipart::client::Multipart`, as well @@ -19,26 +18,26 @@ as integration with the new `hyper::Client` API via `multipart::client::lazy::Mu Server integration for `hyper::server::Request` via `multipart::server::Multipart`. -####[Iron](http://ironframework.io) +### [Iron](http://ironframework.io) via the `iron` feature. Provides regular server-side integration with `iron::Request` via `multipart::server::Multipart`, as well as a convenient `BeforeMiddleware` implementation in `multipart::server::iron::Intercept`. -####[tiny\_http](https://crates.io/crates/tiny_http/) +### [tiny\_http](https://crates.io/crates/tiny_http/) via the `tiny_http` feature. Provides server-side integration with `tiny_http::Request` via `multipart::server::Multipart`. -####[Nickel](http://nickel.rs/) +### [Nickel](http://nickel.rs/) **Note**: Moved to `multipart-nickel` crate, see [nickel/examples/nickel.rs](nickel/examples/nickel.rs) for updated integration example. Provides server-side integration with `&mut nickel::Request` via `multipart::server::Multipart`. -License -------- +## License + Licensed under either of @@ -47,7 +46,7 @@ Licensed under either of at your option. -### Contribution +## Contribution Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any From 16171600fc2cab011930e4447865743d27a0ad03 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Thu, 30 Mar 2017 11:43:59 -0700 Subject: [PATCH 311/453] Bump version, pin minimum Rust at 1.13 --- multipart/.travis.yml | 15 ++++++++++----- multipart/Cargo.toml | 2 +- multipart/README.md | 2 ++ 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/multipart/.travis.yml b/multipart/.travis.yml index 5b57f3127..c34eee55f 100644 --- a/multipart/.travis.yml +++ b/multipart/.travis.yml @@ -5,10 +5,10 @@ branches: except: - fuzzing rust: + - 1.13.0 - stable - beta - # FIXME: when https://github.com/rust-lang/cargo/issues/3844 is closed - # - nightly + - nightly os: - linux # openssl is broken on OS X again @@ -16,7 +16,12 @@ os: env: - RUST_LOG=multipart=info RUST_BACKTRACE=1 script: - - cargo build -v --features all - - cargo test -v --features all - - cargo doc -v --no-deps --features all + - if [ ${TRAVIS_RUST_VERSION} = "nightly" ]; then + cargo build --verbose --features "nightly"; + cargo test --verbose --features "nightly"; + cargo doc --verbose --features "nightly"; + fi + - cargo build --verbose; + - cargo test --verbose; + - cargo doc --verbose; - (cd nickel && cargo test -v) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index f9e29908d..0b819b742 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "multipart" -version = "0.10.2" +version = "0.11.0" authors = ["Austin Bonander "] diff --git a/multipart/README.md b/multipart/README.md index 89e6017d5..6970bbbc0 100644 --- a/multipart/README.md +++ b/multipart/README.md @@ -4,6 +4,8 @@ Client- and server-side abstractions for HTTP file uploads (POST requests with Supports several different HTTP crates. +Minimum supported Rust version: 1.13.0 (for `?` operator) + ### [Documentation](http://docs.rs/multipart/) ## Integrations From 6fb1c7963ca83730fc0893404188b60d43d99c20 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Thu, 30 Mar 2017 11:46:56 -0700 Subject: [PATCH 312/453] Upgrade Hyper --- multipart/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 0b819b742..a62936674 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -30,7 +30,7 @@ safemem = { version = "0.2", optional = true } twoway = { version = "0.1", optional = true } # Optional Integrations -hyper = { version = "0.9", optional = true, default-features = false } +hyper = { version = ">=0.9, <0.10", optional = true, default-features = false } iron = { version = ">=0.4,<0.6", optional = true } # NOTE: use `nickel_` feature, as `hyper` feature is required. From 27b4d606c64b3f4083b43794d4126b6af974178e Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Thu, 30 Mar 2017 11:51:14 -0700 Subject: [PATCH 313/453] Remove in-crate Nickel support --- multipart/Cargo.toml | 6 +-- multipart/examples/nickel.rs | 87 ++-------------------------------- multipart/nickel/Cargo.toml | 9 +++- multipart/src/lib.rs | 9 +--- multipart/src/server/mod.rs | 3 -- multipart/src/server/nickel.rs | 33 ------------- 6 files changed, 13 insertions(+), 134 deletions(-) delete mode 100644 multipart/src/server/nickel.rs diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index a62936674..c71bc7023 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -33,9 +33,6 @@ twoway = { version = "0.1", optional = true } hyper = { version = ">=0.9, <0.10", optional = true, default-features = false } iron = { version = ">=0.4,<0.6", optional = true } -# NOTE: use `nickel_` feature, as `hyper` feature is required. -nickel = { optional = true, version = "0.9" } - tiny_http = { version = "0.5", optional = true } [dev-dependencies] @@ -46,9 +43,8 @@ client = [] default = ["all"] server = ["buf_redux", "httparse", "safemem", "twoway"] mock = [] -nickel_ = ["nickel", "hyper"] nightly = ["clippy"] # Use this to enable SSE4.2 instructions in boundary finding # TODO: Benchmark this sse4 = ["nightly", "twoway/pcmp"] -all = ["client", "server", "hyper", "iron", "nickel_", "tiny_http", "mock"] +all = ["client", "server", "hyper", "iron", "tiny_http", "mock"] diff --git a/multipart/examples/nickel.rs b/multipart/examples/nickel.rs index 576659f7e..78f940567 100644 --- a/multipart/examples/nickel.rs +++ b/multipart/examples/nickel.rs @@ -1,87 +1,8 @@ -//! **Note**: in-crate integration for Nickel is deprecated and will be removed in 0.11.0; +//! **Note**: in-crate integration for Nickel was removed in 0.11.0; //! integration will be provided in the //! [`multipart-nickel`](https://crates.io/crates/multipart-nickel) //! crate for the foreseeable future. -extern crate multipart; -extern crate nickel; +//! +//! Please see `nickel/examples/nickel.rs` for the new integration. -use std::fs::File; -use std::io::Read; -use nickel::{HttpRouter, MiddlewareResult, Nickel, Request, Response}; - -use multipart::server::{Multipart, Entries, SaveResult}; - -fn handle_multipart<'mw>(req: &mut Request, mut res: Response<'mw>) -> MiddlewareResult<'mw> { - match Multipart::from_request(req) { - Ok(mut multipart) => { - match multipart.save().temp() { - SaveResult::Full(entries) => process_entries(res, entries), - - SaveResult::Partial(entries, e) => { - println!("Partial errors ... {:?}", e); - return process_entries(res, entries.keep_partial()); - }, - - SaveResult::Error(e) => { - println!("There are errors in multipart POSTing ... {:?}", e); - res.set(nickel::status::StatusCode::InternalServerError); - return res.send(format!("Server could not handle multipart POST! {:?}", e)); - }, - } - } - Err(_) => { - res.set(nickel::status::StatusCode::BadRequest); - return res.send("Request seems not was a multipart request") - } - } -} - -/// Processes saved entries from multipart request. -/// Returns an OK response or an error. -fn process_entries<'mw>(res: Response<'mw>, entries: Entries) -> MiddlewareResult<'mw> { - for (name, field) in entries.fields { - println!("Field {:?}: {:?}", name, field); - } - - for (name, files) in entries.files { - println!("Field {:?} has {} files:", name, files.len()); - - for saved_file in files { - match File::open(&saved_file.path) { - Ok(mut file) => { - let mut contents = String::new(); - if let Err(e) = file.read_to_string(&mut contents) { - println!("Could not read file {:?}. Error: {:?}", saved_file.filename, e); - return res.error(nickel::status::StatusCode::BadRequest, "The uploaded file was not readable") - } - - println!("File {:?} ({:?}):", saved_file.filename, saved_file.content_type); - println!("{}", contents); - file - } - Err(e) => { - println!("Could open file {:?}. Error: {:?}", saved_file.filename, e); - return res.error(nickel::status::StatusCode::BadRequest, "The uploaded file was not readable") - } - }; - } - } - - res.send("Ok") -} - -fn main() { - let mut srv = Nickel::new(); - - srv.post("/multipart_upload/", handle_multipart); - - // Start this example via: - // - // `cargo run --example nickel --features nickel` - // - // And - if you are in the root of this repository - do an example - // upload via: - // - // `curl -F file=@LICENSE 'http://localhost:6868/multipart_upload/'` - srv.listen("127.0.0.1:6868").expect("Failed to bind server"); -} +fn main() {} diff --git a/multipart/nickel/Cargo.toml b/multipart/nickel/Cargo.toml index cb5ed037d..0a4719552 100644 --- a/multipart/nickel/Cargo.toml +++ b/multipart/nickel/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "multipart-nickel" -version = "0.1.0" +version = "0.2.0" authors = ["Austin Bonander "] description = "Support for `multipart/form-data` bodies in Nickel via the `multipart` crate." @@ -10,5 +10,10 @@ license = "MIT OR Apache-2.0" [dependencies] hyper = "0.9" -multipart = { version = "0.10", default-features = false, features = ["hyper", "server"] } nickel = "0.9" + +[dependencies.multipart] +version = "0.11" +#path = "../" +default-features = false +features = ["hyper", "server"] diff --git a/multipart/src/lib.rs b/multipart/src/lib.rs index 68d6a0b7d..a07b19824 100644 --- a/multipart/src/lib.rs +++ b/multipart/src/lib.rs @@ -25,11 +25,7 @@ //! * `tiny_http`: Integration with the [`tiny_http`](https://github.com/frewsxcv/tiny-http) //! crate. See the [`server::tiny_http`](server/tiny_http/index.html) module for more information. //! -//! * `nickel_`: Integration with the [Nickel](http://nickel.rs) web application framework. -//! See the [`server::nickel`](server/nickel/index.html) module for more information. Enables the `hyper` -//! feature. -//! -//! **Note**: in-crate integration for Nickel is deprecated and will be removed in 0.11.0; +//! **Note**: in-crate integration for Nickel was removed in 0.11.0; //! integration will be provided in the //! [`multipart-nickel`](https://crates.io/crates/multipart-nickel) //! crate for the foreseeable future. @@ -57,9 +53,6 @@ extern crate hyper; #[cfg(feature = "iron")] extern crate iron; -#[cfg(feature = "nickel")] -extern crate nickel; - #[cfg(feature = "tiny_http")] extern crate tiny_http; diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index 1c209f621..007b030ae 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -71,9 +71,6 @@ pub mod hyper; #[cfg(feature = "iron")] pub mod iron; -#[cfg(feature = "nickel")] -pub mod nickel; - #[cfg(feature = "tiny_http")] pub mod tiny_http; diff --git a/multipart/src/server/nickel.rs b/multipart/src/server/nickel.rs deleted file mode 100644 index 16ebe22b3..000000000 --- a/multipart/src/server/nickel.rs +++ /dev/null @@ -1,33 +0,0 @@ -//! Server-side integration with [Nickel](http://nickel.rs/) via the `nickel_` feature -//! (optional, enables `hyper` feature). -//! -//! Not shown here: [`impl HttpRequest for &mut nickel::Request`](../trait.HttpRequest.html#implementors). -//! -//! **Note**: in-crate integration for Nickel is deprecated and will be removed in 0.11.0; -//! integration will be provided in the -//! [`multipart-nickel`](https://crates.io/crates/multipart-nickel) -//! crate for the foreseeable future. -#![deprecated(since = "0.10.2", note = "Nickel integration has moved to the `multipart-nickel` - crate; in-crate integration will be removed in 0.11.0")] -use nickel::Request as NickelRequest; - -use hyper::server::Request as HyperRequest; - -use super::HttpRequest; - -#[deprecated(since = "0.10.2", note = "Nickel integration has moved to the `multipart-nickel` - crate; in-crate integration will be removed in 0.11.0")] -impl<'r, 'mw, 'server, D: 'mw> HttpRequest for &'r mut NickelRequest<'mw, 'server, D> { - type Body = &'r mut HyperRequest<'mw, 'server>; - - fn multipart_boundary(&self) -> Option<&str> { - self.origin.multipart_boundary() - } - - fn body(self) -> Self::Body { - info!("In-crate Nickel integration is deprecated and will be removed in 0.11.0; \ - please consider switching to the `multipart-nickel` crate to avoid breakage."); - - &mut self.origin - } -} From a62f8dfb93a481cef350b6ba0f46a4bbc36c0fc9 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Fri, 31 Mar 2017 16:11:42 -0700 Subject: [PATCH 314/453] Update multipart-fuzz's Cargo.lock --- multipart/fuzz/Cargo.lock | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/multipart/fuzz/Cargo.lock b/multipart/fuzz/Cargo.lock index 5d8ef624a..748229dd6 100644 --- a/multipart/fuzz/Cargo.lock +++ b/multipart/fuzz/Cargo.lock @@ -3,8 +3,8 @@ name = "multipart-fuzz" version = "0.0.1" dependencies = [ "libfuzzer-sys 0.1.0 (git+https://github.com/rust-fuzz/libfuzzer-sys.git)", - "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", - "multipart 0.10.0", + "log 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", + "multipart 0.11.0", ] [[package]] @@ -18,7 +18,7 @@ dependencies = [ [[package]] name = "gcc" -version = "0.3.43" +version = "0.3.45" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -34,14 +34,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "libfuzzer-sys" version = "0.1.0" -source = "git+https://github.com/rust-fuzz/libfuzzer-sys.git#9d00b47e0ef203543d304f2a969a3ced8817369c" +source = "git+https://github.com/rust-fuzz/libfuzzer-sys.git#36a3928eef5c3c38eb0f251962395bb510c39d46" dependencies = [ - "gcc 0.3.43 (registry+https://github.com/rust-lang/crates.io-index)", + "gcc 0.3.45 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "log" -version = "0.3.6" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -54,10 +54,10 @@ dependencies = [ [[package]] name = "mime" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -65,7 +65,7 @@ name = "mime_guess" version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "mime 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "mime 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", "phf 0.7.21 (registry+https://github.com/rust-lang/crates.io-index)", "phf_codegen 0.7.21 (registry+https://github.com/rust-lang/crates.io-index)", "unicase 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -73,12 +73,12 @@ dependencies = [ [[package]] name = "multipart" -version = "0.10.0" +version = "0.11.0" dependencies = [ "buf_redux 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", "httparse 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)", - "mime 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", + "mime 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", "mime_guess 1.8.1 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)", "safemem 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -117,7 +117,7 @@ name = "phf_shared" version = "0.7.21" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "siphasher 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "siphasher 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", "unicase 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -154,7 +154,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "siphasher" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -183,13 +183,13 @@ dependencies = [ [metadata] "checksum buf_redux 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e1497634c131ba13483b6e8123f69e219253b018bb32949eefd55c6b5051585d" -"checksum gcc 0.3.43 (registry+https://github.com/rust-lang/crates.io-index)" = "c07c758b972368e703a562686adb39125707cc1ef3399da8c019fc6c2498a75d" +"checksum gcc 0.3.45 (registry+https://github.com/rust-lang/crates.io-index)" = "40899336fb50db0c78710f53e87afc54d8c7266fb76262fecc78ca1a7f09deae" "checksum httparse 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a6e7a63e511f9edffbab707141fbb8707d1a3098615fb2adbd5769cdfcc9b17d" "checksum libc 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)" = "88ee81885f9f04bff991e306fea7c1c60a5f0f9e409e99f6b40e3311a3363135" "checksum libfuzzer-sys 0.1.0 (git+https://github.com/rust-fuzz/libfuzzer-sys.git)" = "" -"checksum log 0.3.6 (registry+https://github.com/rust-lang/crates.io-index)" = "ab83497bf8bf4ed2a74259c1c802351fcd67a65baa86394b6ba73c36f4838054" +"checksum log 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)" = "5141eca02775a762cc6cd564d8d2c50f67c0ea3a372cbf1c51592b3e029e10ad" "checksum memchr 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)" = "d8b629fb514376c675b98c1421e80b151d3817ac42d7c667717d282761418d20" -"checksum mime 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "b5c93a4bd787ddc6e7833c519b73a50883deb5863d76d9b71eb8216fb7f94e66" +"checksum mime 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "5514f038123342d01ee5f95129e4ef1e0470c93bc29edf058a46f9ee3ba6737e" "checksum mime_guess 1.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "76da6df85047af8c0edfa53f48eb1073012ce1cc95c8fedc0a374f659a89dd65" "checksum phf 0.7.21 (registry+https://github.com/rust-lang/crates.io-index)" = "cb325642290f28ee14d8c6201159949a872f220c62af6e110a56ea914fbe42fc" "checksum phf_codegen 0.7.21 (registry+https://github.com/rust-lang/crates.io-index)" = "d62594c0bb54c464f633175d502038177e90309daf2e0158be42ed5f023ce88f" @@ -200,7 +200,7 @@ dependencies = [ "checksum safemem 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "725b3bf47ae40b4abcd27b5f0a9540369426a29f7b905649b3e1468e13e22009" "checksum safemem 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e27a8b19b835f7aea908818e871f5cc3a5a186550c30773be987e155e8163d8f" "checksum semver 0.1.20 (registry+https://github.com/rust-lang/crates.io-index)" = "d4f410fedcf71af0345d7607d246e7ad15faaadd49d240ee3b24e5dc21a820ac" -"checksum siphasher 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2ffc669b726f2bc9a3bcff66e5e23b56ba6bf70e22a34c3d7b6d0b3450b65b84" +"checksum siphasher 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "0df90a788073e8d0235a67e50441d47db7c8ad9debd91cbf43736a2a92d36537" "checksum tempdir 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "87974a6f5c1dfb344d733055601650059a3363de2a6104819293baff662132d6" "checksum twoway 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "e267e178055eb3b081224bbef62d4f508ae3c9f000b6ae6ccdb04a0d9c34b77f" "checksum unicase 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "13a5906ca2b98c799f4b1ab4557b76367ebd6ae5ef14930ec841c74aed5f3764" From 2c56b67ee4de92627f6ef3240b67c4d181e44b66 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Fri, 31 Mar 2017 17:06:10 -0700 Subject: [PATCH 315/453] Disable logging in fuzzer --- multipart/fuzz/fuzzers/logger.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/fuzz/fuzzers/logger.rs b/multipart/fuzz/fuzzers/logger.rs index 88d2c3909..ee8b5d914 100644 --- a/multipart/fuzz/fuzzers/logger.rs +++ b/multipart/fuzz/fuzzers/logger.rs @@ -2,7 +2,7 @@ extern crate log; use self::log::{LogLevelFilter, Log, LogMetadata, LogRecord}; -const MAX_LOG_LEVEL: LogLevelFilter = LogLevelFilter::Debug; +const MAX_LOG_LEVEL: LogLevelFilter = LogLevelFilter::Off; struct Logger; From 543f5915efd2a44b0463cfca849b8c93357852fb Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Fri, 31 Mar 2017 17:06:20 -0700 Subject: [PATCH 316/453] Fix fuzzing script (cargo-fuzz working directory changed) --- multipart/fuzz_server.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/multipart/fuzz_server.sh b/multipart/fuzz_server.sh index 9d99a0a91..723a5de7d 100755 --- a/multipart/fuzz_server.sh +++ b/multipart/fuzz_server.sh @@ -1,2 +1,3 @@ #! /bin/sh -cargo fuzz run server_basic -- -dict=fuzzer_dict -only_ascii=1 -timeout=60 $@ +# pwd +cargo fuzz run server_basic -- -dict=fuzz/fuzzer_dict -only_ascii=1 -timeout=60 $@ From 02dfb908cbd724d493b045e3951eee24c5eda1c0 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Fri, 31 Mar 2017 17:18:50 -0700 Subject: [PATCH 317/453] Remove clippy from nightly feature --- multipart/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index c71bc7023..165956fd8 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -43,7 +43,7 @@ client = [] default = ["all"] server = ["buf_redux", "httparse", "safemem", "twoway"] mock = [] -nightly = ["clippy"] +nightly = [] # Use this to enable SSE4.2 instructions in boundary finding # TODO: Benchmark this sse4 = ["nightly", "twoway/pcmp"] From e8d4ae4c327ae8b1774474f3575800385cf825f0 Mon Sep 17 00:00:00 2001 From: Armin Ronacher Date: Tue, 18 Apr 2017 03:04:43 +0200 Subject: [PATCH 318/453] Bump hyper to 0.10 (#78) * Changed hyper version range to >=0.9, <0.11 --- multipart/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 165956fd8..373c4cac3 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -30,7 +30,7 @@ safemem = { version = "0.2", optional = true } twoway = { version = "0.1", optional = true } # Optional Integrations -hyper = { version = ">=0.9, <0.10", optional = true, default-features = false } +hyper = { version = ">=0.9, <0.11", optional = true, default-features = false } iron = { version = ">=0.4,<0.6", optional = true } tiny_http = { version = "0.5", optional = true } From 0be1c0e4ace26747f7573b7c099598d88045b2fd Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Wed, 19 Apr 2017 23:35:35 -0700 Subject: [PATCH 319/453] Allow setting fuzz data length by env var --- multipart/fuzz_server.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/fuzz_server.sh b/multipart/fuzz_server.sh index 723a5de7d..b4878f5a7 100755 --- a/multipart/fuzz_server.sh +++ b/multipart/fuzz_server.sh @@ -1,3 +1,3 @@ #! /bin/sh # pwd -cargo fuzz run server_basic -- -dict=fuzz/fuzzer_dict -only_ascii=1 -timeout=60 $@ +cargo fuzz run server_basic -- -dict=fuzz/fuzzer_dict -only_ascii=1 -timeout=60 ${FUZZ_LEN:+ -max_len=$FUZZ_LEN} $@ From 776364f605499058a19d9649dbc6ce63db5ca6e6 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Mon, 17 Apr 2017 18:23:09 -0700 Subject: [PATCH 320/453] Bump `multipart`, `multipart-nickel` versions --- multipart/Cargo.toml | 3 +-- multipart/nickel/Cargo.toml | 7 ++++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 373c4cac3..0c92c2029 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "multipart" -version = "0.11.0" +version = "0.12.0" authors = ["Austin Bonander "] @@ -32,7 +32,6 @@ twoway = { version = "0.1", optional = true } # Optional Integrations hyper = { version = ">=0.9, <0.11", optional = true, default-features = false } iron = { version = ">=0.4,<0.6", optional = true } - tiny_http = { version = "0.5", optional = true } [dev-dependencies] diff --git a/multipart/nickel/Cargo.toml b/multipart/nickel/Cargo.toml index 0a4719552..5688bb328 100644 --- a/multipart/nickel/Cargo.toml +++ b/multipart/nickel/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "multipart-nickel" -version = "0.2.0" +version = "0.3.0" authors = ["Austin Bonander "] description = "Support for `multipart/form-data` bodies in Nickel via the `multipart` crate." @@ -13,7 +13,8 @@ hyper = "0.9" nickel = "0.9" [dependencies.multipart] -version = "0.11" -#path = "../" +version = "0.12" +# I had no idea you could also specify a path +path = "../" default-features = false features = ["hyper", "server"] From 668e93a31ad2a7ede0376be63136c888885b4a88 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Mon, 24 Apr 2017 20:50:30 -0700 Subject: [PATCH 321/453] Logging macros don't evaluate their arguments if the log level isn't enabled --- multipart/src/server/boundary.rs | 38 ++++++++++---------------------- 1 file changed, 12 insertions(+), 26 deletions(-) diff --git a/multipart/src/server/boundary.rs b/multipart/src/server/boundary.rs index 1d4ecfd3d..91423c5f9 100644 --- a/multipart/src/server/boundary.rs +++ b/multipart/src/server/boundary.rs @@ -55,14 +55,10 @@ impl BoundaryReader where R: Read { return Ok(buf); } - if log_enabled!(LogLevel::Trace) { - trace!("Buf: {:?}", String::from_utf8_lossy(buf)); - } + trace!("Buf: {:?}", String::from_utf8_lossy(buf)); - debug!( - "Before-loop Buf len: {} Search idx: {} Boundary read: {:?}", - buf.len(), self.search_idx, self.boundary_read - ); + debug!("Before-loop Buf len: {} Search idx: {} Boundary read: {:?}", + buf.len(), self.search_idx, self.boundary_read); if !self.boundary_read && self.search_idx < buf.len() { let lookahead = &buf[self.search_idx..]; @@ -81,17 +77,15 @@ impl BoundaryReader where R: Read { } } - debug!( - "After-loop Buf len: {} Search idx: {} Boundary read: {:?}", - buf.len(), self.search_idx, self.boundary_read - ); + debug!("After-loop Buf len: {} Search idx: {} Boundary read: {:?}", + buf.len(), self.search_idx, self.boundary_read); // If the two bytes before the boundary are a CR-LF, we need to back up // the cursor so we don't yield bytes that client code isn't expecting. if self.boundary_read && self.search_idx >= 2 { let two_bytes_before = &buf[self.search_idx - 2 .. self.search_idx]; - debug!("Two bytes before: {:?} ({:?}) (\"\\r\\n\": {:?})", + trace!("Two bytes before: {:?} ({:?}) (\"\\r\\n\": {:?})", String::from_utf8_lossy(two_bytes_before), two_bytes_before, b"\r\n"); if two_bytes_before == &*b"\r\n" { @@ -102,9 +96,7 @@ impl BoundaryReader where R: Read { let ret_buf = &buf[..self.search_idx]; - if log_enabled!(LogLevel::Trace) { - trace!("Returning buf: {:?}", String::from_utf8_lossy(ret_buf)); - } + trace!("Returning buf: {:?}", String::from_utf8_lossy(ret_buf)); Ok(ret_buf) } @@ -132,10 +124,8 @@ impl BoundaryReader where R: Read { self.source.consume(self.search_idx); self.search_idx = 0; - if log_enabled!(LogLevel::Trace) { - trace!("Consumed up to self.search_idx, remaining buf: {:?}", - String::from_utf8_lossy(self.source.get_buf())); - } + trace!("Consumed up to self.search_idx, remaining buf: {:?}", + String::from_utf8_lossy(self.source.get_buf())); let consume_amt = { let buf = self.source.get_buf(); @@ -156,15 +146,11 @@ impl BoundaryReader where R: Read { if bytes_after == *b"--" { self.at_end = true; } else if bytes_after != *b"\r\n" { - warn!("Unexpected bytes following boundary: {:?}", String::from_utf8_lossy(&bytes_after)); + debug!("Unexpected bytes following boundary: {:?}", String::from_utf8_lossy(&bytes_after)); } - if log_enabled!(LogLevel::Trace) { - trace!("Consumed boundary (at_end: {:?}), remaining buf: {:?}", - self.at_end, - String::from_utf8_lossy(self.source.get_buf()) - ); - } + trace!("Consumed boundary (at_end: {:?}), remaining buf: {:?}", self.at_end, + String::from_utf8_lossy(self.source.get_buf())); Ok(self.at_end) } From 215f8579ec599d54c44c3ce2efd709d42a75f2d4 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Mon, 24 Apr 2017 22:12:16 -0700 Subject: [PATCH 322/453] Cleanup formatting --- multipart/src/server/boundary.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/multipart/src/server/boundary.rs b/multipart/src/server/boundary.rs index 91423c5f9..5c68abcb7 100644 --- a/multipart/src/server/boundary.rs +++ b/multipart/src/server/boundary.rs @@ -211,11 +211,11 @@ mod test { use std::io::prelude::*; const BOUNDARY: &'static str = "boundary"; - const TEST_VAL: &'static str = "--boundary\r -dashed-value-1\r ---boundary\r -dashed-value-2\r ---boundary--"; + const TEST_VAL: &'static str = "--boundary\r\n\ + dashed-value-1\r\n\ + --boundary\r\n\ + dashed-value-2\r\n\ + --boundary--"; #[test] fn test_boundary() { From a90b1ff3beabf14a9b26cb8bd4faef877294f409 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sat, 3 Jun 2017 15:51:27 -0700 Subject: [PATCH 323/453] Add basic benchmark --- multipart/Cargo.toml | 1 + multipart/src/lib.rs | 1 + multipart/src/server/boundary.rs | 50 ++++++++++++++++++++++++++++---- 3 files changed, 46 insertions(+), 6 deletions(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 0c92c2029..d41a79503 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -43,6 +43,7 @@ default = ["all"] server = ["buf_redux", "httparse", "safemem", "twoway"] mock = [] nightly = [] +bench = [] # Use this to enable SSE4.2 instructions in boundary finding # TODO: Benchmark this sse4 = ["nightly", "twoway/pcmp"] diff --git a/multipart/src/lib.rs b/multipart/src/lib.rs index a07b19824..02921c6e2 100644 --- a/multipart/src/lib.rs +++ b/multipart/src/lib.rs @@ -32,6 +32,7 @@ #![cfg_attr(feature="clippy", feature(plugin))] #![cfg_attr(feature="clippy", plugin(clippy))] #![cfg_attr(feature="clippy", deny(clippy))] +#![cfg_attr(feature = "bench", feature(test))] #![deny(missing_docs)] #[macro_use] diff --git a/multipart/src/server/boundary.rs b/multipart/src/server/boundary.rs index 5c68abcb7..05bae8152 100644 --- a/multipart/src/server/boundary.rs +++ b/multipart/src/server/boundary.rs @@ -156,6 +156,21 @@ impl BoundaryReader where R: Read { } } +#[cfg(feature = "bench")] +impl<'a> BoundaryReader> { + fn new_with_bytes(bytes: &'a [u8], boundary: &str) -> Self { + Self::from_reader(io::Cursor::new(bytes), boundary) + } + + fn reset(&mut self) { + // Dump buffer and reset cursor + self.source.seek(io::SeekFrom::Start(0)); + self.at_end = false; + self.boundary_read = false; + self.search_idx = 0; + } +} + impl Borrow for BoundaryReader { fn borrow(&self) -> &R { self.source.get_ref() @@ -223,9 +238,11 @@ mod test { debug!("Testing boundary (no split)"); let src = &mut TEST_VAL.as_bytes(); - let reader = BoundaryReader::from_reader(src, BOUNDARY); + let mut reader = BoundaryReader::from_reader(src, BOUNDARY); + + let mut buf = String::new(); - test_boundary_reader(reader); + test_boundary_reader(&mut reader, &mut buf); } struct SplitReader<'a> { @@ -264,20 +281,22 @@ mod test { fn test_split_boundary() { let _ = ::env_logger::init(); debug!("Testing boundary (split)"); + + let mut buf = String::new(); // Substitute for `.step_by()` being unstable. for split_at in 0 .. TEST_VAL.len(){ debug!("Testing split at: {}", split_at); let src = SplitReader::split(TEST_VAL.as_bytes(), split_at); - let reader = BoundaryReader::from_reader(src, BOUNDARY); - test_boundary_reader(reader); + let mut reader = BoundaryReader::from_reader(src, BOUNDARY); + test_boundary_reader(&mut reader, &mut buf); } } - fn test_boundary_reader(mut reader: BoundaryReader) { - let ref mut buf = String::new(); + fn test_boundary_reader(reader: &mut BoundaryReader, buf: &mut String) { + buf.clear(); debug!("Read 1"); let _ = reader.read_to_string(buf).unwrap(); @@ -307,4 +326,23 @@ mod test { let _ = reader.read_to_string(buf).unwrap(); assert_eq!(buf, ""); } + + #[cfg(feature = "bench")] + mod bench { + extern crate test; + use self::test::Bencher; + + use super::*; + + #[bench] + fn bench_boundary_reader(b: &mut Bencher) { + let mut reader = BoundaryReader::new_with_bytes(TEST_VAL.as_bytes(), BOUNDARY); + let mut buf = String::with_capacity(256); + + b.iter(|| { + reader.reset(); + test_boundary_reader(&mut reader, &mut buf); + }); + } + } } From acbfdc28b74ce3a1527693be9f37956c6c40e7bb Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sun, 4 Jun 2017 15:20:39 -0700 Subject: [PATCH 324/453] lazy::PreparedFields: ditch internal buffering for a simpler implementation --- multipart/Cargo.toml | 2 +- multipart/src/client/lazy.rs | 276 ++++++++++++++++------------------- multipart/src/lib.rs | 2 + multipart/src/server/mod.rs | 1 - 4 files changed, 126 insertions(+), 155 deletions(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index d41a79503..0dac58489 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -20,13 +20,13 @@ log = "0.3" mime = "0.2" mime_guess = "1.8" rand = "0.3" +safemem = { version = "0.2", optional = true } tempdir = ">=0.3.4" clippy = { version = ">=0.0, <0.1", optional = true} #Server Dependencies buf_redux = { version = "0.6", optional = true } httparse = { version = "1.2", optional = true } -safemem = { version = "0.2", optional = true } twoway = { version = "0.1", optional = true } # Optional Integrations diff --git a/multipart/src/client/lazy.rs b/multipart/src/client/lazy.rs index df53ec165..39ebcc60f 100644 --- a/multipart/src/client/lazy.rs +++ b/multipart/src/client/lazy.rs @@ -11,7 +11,7 @@ use std::path::{Path, PathBuf}; use std::io::prelude::*; use std::io::Cursor; -use std::{fmt, io, mem, vec}; +use std::{fmt, io, mem}; use super::{HttpRequest, HttpStream, MultipartWriter}; @@ -107,7 +107,7 @@ impl<'a, E: fmt::Display> fmt::Display for LazyError<'a, E> { /// /// Sacrifices static dispatch for support for dynamic construction. Reusable. /// -/// ####Lifetimes +/// #### Lifetimes /// * `'n`: Lifetime for field **n**ames; will only escape this struct in `LazyIoError<'n>`. /// * `'d`: Lifetime for **d**ata: will only escape this struct in `PreparedFields<'d>`. #[derive(Debug, Default)] @@ -154,7 +154,7 @@ impl<'n, 'd> Multipart<'n, 'd> { Field { name: name.into(), data: Data::Stream(Stream { - content_type: mime, + content_type: mime.unwrap_or_else(::mime_guess::octet_stream), filename: filename.map(|f| f.into()), stream: Box::new(stream) }), @@ -166,60 +166,35 @@ impl<'n, 'd> Multipart<'n, 'd> { /// Convert `req` to `HttpStream`, write out the fields in this request, and finish the /// request, returning the response if successful, or the first error encountered. - pub fn send(&mut self, req: R) -> Result<< R::Stream as HttpStream >::Response, LazyError<'n, < R::Stream as HttpStream >::Error>> { - let (boundary, stream) = try_lazy!(super::open_stream(req, None)); - let mut writer = MultipartWriter::new(stream, boundary); + /// + /// If any files were added by path they will now be opened for reading. + pub fn send(&mut self, mut req: R) -> Result<< R::Stream as HttpStream >::Response, LazyError<'n, < R::Stream as HttpStream >::Error>> { + let mut prepared = try_lazy!(self.prepare()); - for mut field in self.fields.drain(..) { - try_lazy!(field.name, field.write_out(&mut writer)); - } + req.apply_headers(prepared.boundary(), prepared.content_len()); + + let mut stream = try_lazy!(req.open_stream()); + + try_lazy!(io::copy(&mut prepared, &mut stream)); - try_lazy!(writer.finish()).finish().map_err(LazyError::without_field) + stream.finish().map_err(LazyError::without_field) } /// Export the multipart data contained in this lazy request as an adaptor which implements `Read`. /// - /// A certain amount of field data will be buffered. See - /// [`prepare_threshold()`](#method.prepare_threshold) for more information on this behavior. + /// During this step, if any files were added by path then they will be opened for reading + /// and their length measured. pub fn prepare(&mut self) -> LazyIoResult<'n, PreparedFields<'d>> { - self.prepare_threshold(Some(DEFAULT_BUFFER_THRESHOLD)) - } - - /// Export the multipart data contained in this lazy request to an adaptor which implements `Read`. - /// - /// #### Buffering - /// For efficiency, text and file fields smaller than `buffer_threshold` are copied to an in-memory buffer. If `None`, - /// all fields are copied to memory. - pub fn prepare_threshold(&mut self, buffer_threshold: Option) -> LazyIoResult<'n, PreparedFields<'d>> { - let boundary = super::gen_boundary(); - PreparedFields::from_fields(&mut self.fields, boundary.into(), buffer_threshold) + PreparedFields::from_fields(&mut self.fields) } } -const DEFAULT_BUFFER_THRESHOLD: u64 = 8 * 1024; - #[derive(Debug)] struct Field<'n, 'd> { name: Cow<'n, str>, data: Data<'n, 'd>, } -impl<'n, 'd> Field<'n, 'd> { - fn write_out(&mut self, writer: &mut MultipartWriter) -> io::Result<()> { - match self.data { - Data::Text(ref text) => writer.write_text(&self.name, text), - Data::File(ref path) => writer.write_file(&self.name, path), - Data::Stream(ref mut stream) => - writer.write_stream( - &mut stream.stream, - &self.name, - stream.filename.as_ref().map(|f| &**f), - stream.content_type.clone(), - ), - } - } -} - enum Data<'n, 'd> { Text(Cow<'d, str>), File(Cow<'d, Path>), @@ -231,60 +206,80 @@ impl<'n, 'd> fmt::Debug for Data<'n, 'd> { match *self { Data::Text(ref text) => write!(f, "Data::Text({:?})", text), Data::File(ref path) => write!(f, "Data::File({:?})", path), - Data::Stream(_) => f.write_str("Data::Stream(Box"), + Data::Stream(_) => f.write_str("Data::Stream(Box)"), } } } struct Stream<'n, 'd> { filename: Option>, - content_type: Option, + content_type: Mime, stream: Box, } -/// The result of [`Multipart::prepare()`](struct.Multipart.html#method.prepare) or -/// `Multipart::prepare_threshold()`. Implements `Read`, contains the entire request body. +/// The result of [`Multipart::prepare()`](struct.Multipart.html#method.prepare). +/// +/// Implements `Read`, contains the entire request body. +/// +/// Individual files/streams are dropped as they are read to completion. +/// +/// ### Note +/// The fields in the request may have been reordered to simplify the preparation step. +/// No compliant server implementation will be relying on the specific ordering of fields anyways. pub struct PreparedFields<'d> { - fields: vec::IntoIter>, - next_field: Option>, - boundary: String, + text_data: Cursor>, + streams: Vec>, + end_boundary: Cursor, content_len: Option, } impl<'d> PreparedFields<'d> { - fn from_fields<'n>(fields: &mut Vec>, boundary: String, buffer_threshold: Option) -> Result> { - let buffer_threshold = buffer_threshold.unwrap_or(::std::u64::MAX); - + fn from_fields<'n>(fields: &mut Vec>) -> Result> { debug!("Field count: {}", fields.len()); - let mut prep_fields = Vec::new(); - - let mut fields = fields.drain(..).peekable(); - - { - let mut writer = MultipartWriter::new(Vec::new(), &*boundary); + // One of the two RFCs specifies that any bytes before the first boundary are to be + // ignored anyway + let mut boundary = format!("\r\n--{}", super::gen_boundary()); + + let mut text_data = Vec::new(); + let mut streams = Vec::new(); + let mut content_len = 0u64; + let mut use_len = true; + + for field in fields.drain(..) { + match field.data { + Data::Text(text) => write!(text_data, "{}\r\nContent-Disposition: form-data; \ + name=\"{}\"\r\n\r\n{}", + boundary, field.name, text).unwrap(), + Data::File(file) => { + let (stream, len) = PreparedField::from_path(field.name, &file, &boundary)?; + content_len += len; + streams.push(stream); + }, + Data::Stream(stream) => { + use_len = false; - while fields.peek().is_some() { - if let Some(rem) = write_fields(&mut writer, &mut fields, buffer_threshold)? { - let contiguous = mem::replace(writer.inner_mut(), Vec::new()); - prep_fields.push(PreparedField::Partial(Cursor::new(contiguous), rem)); - } + streams.push( + PreparedField::from_stream(field.name, &boundary, stream.content_type, + stream.filename, stream.stream)); + }, } - - let contiguous = writer.finish().unwrap(); - - prep_fields.push(PreparedField::Contiguous(Cursor::new(contiguous))); } - let content_len = get_content_len(&prep_fields); + // So we don't write a spurious end boundary + if text_data.is_empty() && streams.is_empty() { + *boundary = String::new(); + } else { + boundary.push_str("--"); + } - debug!("Prepared fields len: {}", prep_fields.len()); + content_len += boundary.len(); Ok(PreparedFields { - fields: prep_fields.into_iter(), - next_field: None, - boundary: boundary, - content_len: content_len, + text_data: Cursor::new(text_data), + streams: streams, + end_boundary: Cursor::new(boundary), + content_len: if use_len { Some(content_len) } else { None } , }) } @@ -296,7 +291,10 @@ impl<'d> PreparedFields<'d> { /// Get the boundary that was used to serialize the request. pub fn boundary(&self) -> &str { - &self.boundary + let boundary = self.end_boundary.get_ref(); + + // Get just the bare boundary string + boundary[4 .. boundary.len() - 2] } } @@ -310,113 +308,81 @@ impl<'d> Read for PreparedFields<'d> { let mut total_read = 0; while total_read < buf.len() { - if self.next_field.is_none() { - self.next_field = self.fields.next(); - } - let buf = &mut buf[total_read..]; - let num_read = if let Some(ref mut field) = self.next_field { - field.read(buf)? + total_read += if !cursor_at_end(&self.text_data) { + self.text_data.read(buf)? + } else if let Some(mut field) = self.streams.pop() { + match field.read(buf) { + Ok(0) => continue, + res => { + self.streams.push(field); + res + }, + }? } else { - break; + self.end_boundary.read(buf)? }; - - if num_read == 0 { - self.next_field = None; - } - - total_read += num_read; } Ok(total_read) } } -fn get_content_len(fields: &[PreparedField]) -> Option { - let mut content_len = 0; +struct PreparedField<'d> { + header: Cursor>, + stream: Box, +} - for field in fields { - match *field { - PreparedField::Contiguous(ref vec) => content_len += vec.get_ref().len() as u64, - PreparedField::Partial(..) => return None, - } +impl<'d> PreparedField<'d> { + fn from_path<'n>(name: &'n str, path: &Path, boundary: &str) -> Result<(Self, u64), LazyIoError<'n>> { + let (content_type, filename) = super::mime_filename(path); + + let file = try_lazy!(name, File::open(path)); + let content_len = try_lazy!(name, file.metadata()).len(); + + let stream = Self::from_stream(name, boundary, content_type, filename, Box::new(file)); + + let content_len = content_len + stream.header.get_ref().len(); + + Ok((stream, content_len)) } - Some(content_len) -} + fn from_stream(name: &str, boundary: &str, content_type: Mime, filename: Option<&str>, stream: Box) -> Self { + let mut header = Vec::new(); -fn write_fields<'n, 'd, I: Iterator>>(writer: &mut MultipartWriter>, fields: I, buffer_threshold: u64) - -> LazyIoResult<'n, Option>> { - for field in fields { - match field.data { - Data::Text(text) => if text.len() as u64 <= buffer_threshold { - try_lazy!(field.name, writer.write_text(&field.name, &*text)); - } else { - try_lazy!(field.name, writer.write_field_headers(&field.name, None, None)); - return Ok(Some(Box::new(io::Cursor::new(CowStrAsRef(text))))); - }, - Data::File(path) => { - let (content_type, filename) = super::mime_filename(&*path); - let mut file = try_lazy!(field.name, File::open(&*path)); - let len = try_lazy!(field.name, file.metadata()).len(); - - if len <= buffer_threshold { - try_lazy!(field.name, writer.write_stream(&mut file, &field.name, filename, Some(content_type))); - } else { - return Ok(Some(Box::new(file))); - } - }, - Data::Stream(stream) => { - let content_type = stream.content_type.or_else(|| Some(::mime_guess::octet_stream())); - let filename = stream.filename.as_ref().map(|f| &**f); - try_lazy!(field.name, writer.write_field_headers(&field.name, filename, content_type)); - return Ok(Some(stream.stream)); - }, + write!(header, "{}\r\nContent-Disposition: form-data; name=\"{}\"", + boundary, name).unwrap(); + + if let Some(filename) = filename { + write!(header, "; filename=\"{}\"").unwrap(); } - } - Ok(None) -} + write!(header, "Content-Type: {}\r\n\r\n", content_type).unwrap(); -#[doc(hidden)] -pub enum PreparedField<'d> { - Contiguous(io::Cursor>), - Partial(io::Cursor>, Box), + PreparedField { + header: Cursor::new(header), + stream: stream + } + } } impl<'d> Read for PreparedField<'d> { fn read(&mut self, buf: &mut [u8]) -> io::Result { - match *self { - PreparedField::Contiguous(ref mut vec) => vec.read(buf), - PreparedField::Partial(ref mut cursor, ref mut remainder) => { - match cursor.read(buf) { - Ok(0) => remainder.read(buf), - res => res, - } - }, + if !cursor_at_end(&self.header) { + self.header.read(buf) + } else { + self.stream.read(buf) } } } impl<'d> fmt::Debug for PreparedField<'d> { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - match *self { - PreparedField::Contiguous(ref bytes) => { - if log_enabled!(LogLevel::Trace) { - write!(f, "PreparedField::Contiguous(\n{:?}\n)", String::from_utf8_lossy(bytes.get_ref())) - } else { - write!(f, "PreparedField::Contiguous(len: {})", bytes.get_ref().len()) - } - }, - PreparedField::Partial(ref bytes, _) => { - if log_enabled!(LogLevel::Trace) { - write!(f, "PreparedField::Partial(\n{:?}\n, Box)", String::from_utf8_lossy(bytes.get_ref())) - } else { - write!(f, "PreparedField::Partial(len: {}, Box)", bytes.get_ref().len()) - } - } - } + f.debug_struct("PreparedField") + .field("header", &self.header) + .field("stream", &"Box") + .finish() } } @@ -465,6 +431,10 @@ impl<'a> IntoCowPath<'a> for &'a str { } } +fn cursor_at_end>(cursor: &Cursor) -> bool { + cursor.position() == (cursor.get_ref().as_ref().len() as u64) +} + #[cfg(feature = "hyper")] mod hyper { use hyper::client::{Body, Client, IntoUrl, RequestBuilder, Response}; diff --git a/multipart/src/lib.rs b/multipart/src/lib.rs index 02921c6e2..53aaa044a 100644 --- a/multipart/src/lib.rs +++ b/multipart/src/lib.rs @@ -48,6 +48,8 @@ extern crate mime_guess; extern crate rand; extern crate tempdir; +extern crate safemem; + #[cfg(feature = "hyper")] extern crate hyper; diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index 007b030ae..acaa8e670 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -13,7 +13,6 @@ extern crate buf_redux; extern crate httparse; -extern crate safemem; extern crate twoway; use std::borrow::Borrow; From 794f38542e78830823087c97641c8fe48a019e0d Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sun, 4 Jun 2017 20:46:52 -0700 Subject: [PATCH 325/453] Fix errors after refactor of lazy.rs --- multipart/src/client/lazy.rs | 39 ++++++++++++++++++++------------ multipart/src/local_test.rs | 2 +- multipart/src/server/boundary.rs | 4 +++- 3 files changed, 29 insertions(+), 16 deletions(-) diff --git a/multipart/src/client/lazy.rs b/multipart/src/client/lazy.rs index 39ebcc60f..2eef46caf 100644 --- a/multipart/src/client/lazy.rs +++ b/multipart/src/client/lazy.rs @@ -19,7 +19,7 @@ macro_rules! try_lazy ( ($field:expr, $try:expr) => ( match $try { Ok(ok) => ok, - Err(e) => return Err(LazyError::with_field($field, e)), + Err(e) => return Err(LazyError::with_field($field.into(), e)), } ); ($try:expr) => ( @@ -64,6 +64,14 @@ impl<'a, E> LazyError<'a, E> { _priv: (), } } + + fn transform_err>(self) -> LazyError<'a, E_> { + LazyError { + field_name: self.field_name, + error: self.error.into(), + _priv: (), + } + } } /// Take `self.error`, discarding `self.field_name`. @@ -169,7 +177,7 @@ impl<'n, 'd> Multipart<'n, 'd> { /// /// If any files were added by path they will now be opened for reading. pub fn send(&mut self, mut req: R) -> Result<< R::Stream as HttpStream >::Response, LazyError<'n, < R::Stream as HttpStream >::Error>> { - let mut prepared = try_lazy!(self.prepare()); + let mut prepared = self.prepare().map_err(LazyError::transform_err)?; req.apply_headers(prepared.boundary(), prepared.content_len()); @@ -260,20 +268,21 @@ impl<'d> PreparedFields<'d> { use_len = false; streams.push( - PreparedField::from_stream(field.name, &boundary, stream.content_type, - stream.filename, stream.stream)); + PreparedField::from_stream(&field.name, &boundary, stream.content_type, + stream.filename.as_ref().map(|f| &**f), + stream.stream)); }, } } // So we don't write a spurious end boundary if text_data.is_empty() && streams.is_empty() { - *boundary = String::new(); + boundary = String::new(); } else { boundary.push_str("--"); } - content_len += boundary.len(); + content_len += boundary.len() as u64; Ok(PreparedFields { text_data: Cursor::new(text_data), @@ -294,7 +303,7 @@ impl<'d> PreparedFields<'d> { let boundary = self.end_boundary.get_ref(); // Get just the bare boundary string - boundary[4 .. boundary.len() - 2] + &boundary[4 .. boundary.len() - 2] } } @@ -307,7 +316,7 @@ impl<'d> Read for PreparedFields<'d> { let mut total_read = 0; - while total_read < buf.len() { + while total_read < buf.len() && !cursor_at_end(&self.end_boundary){ let buf = &mut buf[total_read..]; total_read += if !cursor_at_end(&self.text_data) { @@ -335,15 +344,15 @@ struct PreparedField<'d> { } impl<'d> PreparedField<'d> { - fn from_path<'n>(name: &'n str, path: &Path, boundary: &str) -> Result<(Self, u64), LazyIoError<'n>> { - let (content_type, filename) = super::mime_filename(path); + fn from_path<'n>(name: Cow<'n, str>, path: &Path, boundary: &str) -> Result<(Self, u64), LazyIoError<'n>> { + let (content_type, filename) = super::mime_filename(&path); let file = try_lazy!(name, File::open(path)); let content_len = try_lazy!(name, file.metadata()).len(); - let stream = Self::from_stream(name, boundary, content_type, filename, Box::new(file)); + let stream = Self::from_stream(&name, boundary, content_type, filename, Box::new(file)); - let content_len = content_len + stream.header.get_ref().len(); + let content_len = content_len + (stream.header.get_ref().len() as u64); Ok((stream, content_len)) } @@ -355,10 +364,10 @@ impl<'d> PreparedField<'d> { boundary, name).unwrap(); if let Some(filename) = filename { - write!(header, "; filename=\"{}\"").unwrap(); + write!(header, "; filename=\"{}\"", filename).unwrap(); } - write!(header, "Content-Type: {}\r\n\r\n", content_type).unwrap(); + write!(header, "\r\nContent-Type: {}\r\n\r\n", content_type).unwrap(); PreparedField { header: Cursor::new(header), @@ -369,6 +378,8 @@ impl<'d> PreparedField<'d> { impl<'d> Read for PreparedField<'d> { fn read(&mut self, buf: &mut [u8]) -> io::Result { + debug!("PreparedField::read()"); + if !cursor_at_end(&self.header) { self.header.read(buf) } else { diff --git a/multipart/src/local_test.rs b/multipart/src/local_test.rs index bd8ef261e..bcecfa680 100644 --- a/multipart/src/local_test.rs +++ b/multipart/src/local_test.rs @@ -326,7 +326,7 @@ fn test_client_lazy(test_fields: &TestFields) -> HttpBuffer { } } - let mut prepared = multipart.prepare_threshold(None).unwrap(); + let mut prepared = multipart.prepare().unwrap(); let mut buf = Vec::new(); diff --git a/multipart/src/server/boundary.rs b/multipart/src/server/boundary.rs index 05bae8152..09533663d 100644 --- a/multipart/src/server/boundary.rs +++ b/multipart/src/server/boundary.rs @@ -7,8 +7,10 @@ //! Boundary parsing for `multipart` requests. +use ::safemem; + use super::buf_redux::BufReader; -use super::{safemem, twoway}; +use super::twoway; use log::LogLevel; From 2a68eeda504f152493496d82163724f93ca8974d Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Mon, 5 Jun 2017 02:17:04 -0700 Subject: [PATCH 326/453] Bump min Rust to 1.14 --- multipart/.travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/.travis.yml b/multipart/.travis.yml index c34eee55f..a22644880 100644 --- a/multipart/.travis.yml +++ b/multipart/.travis.yml @@ -5,7 +5,7 @@ branches: except: - fuzzing rust: - - 1.13.0 + - 1.14.0 - stable - beta - nightly From dd63b6557adab1ce3dec275d639288e7d419009c Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sun, 11 Jun 2017 00:56:05 -0700 Subject: [PATCH 327/453] Update Cargo versions --- multipart/Cargo.toml | 4 ++-- multipart/nickel/Cargo.toml | 9 +++++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 0dac58489..bfd906560 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -1,11 +1,11 @@ [package] name = "multipart" -version = "0.12.0" +version = "0.13.0" authors = ["Austin Bonander "] -description = "A backend-agnostic extension for HTTP libraries that provides support for POST multipart/form-data requests on for both client and server." +description = "A backend-agnostic extension for HTTP libraries that provides support for POST multipart/form-data requests on both client and server." keywords = ["form-data", "hyper", "iron", "http", "upload"] diff --git a/multipart/nickel/Cargo.toml b/multipart/nickel/Cargo.toml index 5688bb328..635673bd6 100644 --- a/multipart/nickel/Cargo.toml +++ b/multipart/nickel/Cargo.toml @@ -9,12 +9,13 @@ repository = "https://github.com/abonander/multipart" license = "MIT OR Apache-2.0" [dependencies] -hyper = "0.9" -nickel = "0.9" +hyper = "0.10" +nickel = "0.10" [dependencies.multipart] -version = "0.12" -# I had no idea you could also specify a path +version = "0.13" +# You can specify a path alongside the version so it can be tested against the latest code but published +# against the released version path = "../" default-features = false features = ["hyper", "server"] From c2739be92c57f135a9fa0a9e4e7aa5f2c50f261d Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sun, 11 Jun 2017 01:51:14 -0700 Subject: [PATCH 328/453] Bump minimum Rust version to 1.15 (fixes build) --- multipart/.travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/.travis.yml b/multipart/.travis.yml index a22644880..ad7776428 100644 --- a/multipart/.travis.yml +++ b/multipart/.travis.yml @@ -5,7 +5,7 @@ branches: except: - fuzzing rust: - - 1.14.0 + - 1.15.0 - stable - beta - nightly From c5aac625ea28101770f4ecce6bfc7e1fbe12c359 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Wed, 14 Jun 2017 20:37:16 -0700 Subject: [PATCH 329/453] Fix headers in doc-comments --- multipart/src/client/hyper.rs | 6 +++--- multipart/src/client/mod.rs | 8 ++++---- multipart/src/lib.rs | 2 +- multipart/src/mock.rs | 2 +- multipart/src/server/mod.rs | 2 +- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/multipart/src/client/hyper.rs b/multipart/src/client/hyper.rs index 487191676..6172f96f7 100644 --- a/multipart/src/client/hyper.rs +++ b/multipart/src/client/hyper.rs @@ -24,12 +24,12 @@ use mime::{Mime, TopLevel, SubLevel, Attr, Value}; use super::{HttpRequest, HttpStream}; -/// ####Feature: `hyper` +/// #### Feature: `hyper` impl HttpRequest for Request { type Stream = Request; type Error = HyperError; - /// #Panics + /// # Panics /// If `self.method() != Method::Post`. fn apply_headers(&mut self, boundary: &str, content_len: Option) -> bool { if self.method() != Method::Post { @@ -59,7 +59,7 @@ impl HttpRequest for Request { } } -/// ####Feature: `hyper` +/// #### Feature: `hyper` impl HttpStream for Request { type Request = Request; type Response = Response; diff --git a/multipart/src/client/mod.rs b/multipart/src/client/mod.rs index 89cab11ad..afee0c048 100644 --- a/multipart/src/client/mod.rs +++ b/multipart/src/client/mod.rs @@ -63,7 +63,7 @@ impl Multipart { /// Write a text field to this multipart request. /// `name` and `val` can be either owned `String` or `&str`. /// - /// ##Errors + /// ## Errors /// If something went wrong with the HTTP stream. pub fn write_text, V: AsRef>(&mut self, name: N, val: V) -> Result<&mut Self, S::Error> { map_self!(self, self.writer.write_text(name.as_ref(), val.as_ref())) @@ -77,7 +77,7 @@ impl Multipart { /// /// `name` can be either `String` or `&str`, and `path` can be `PathBuf` or `&Path`. /// - /// ##Errors + /// ## Errors /// If there was a problem opening the file (was a directory or didn't exist), /// or if something went wrong with the HTTP stream. pub fn write_file, P: AsRef>(&mut self, name: N, path: P) -> Result<&mut Self, S::Error> { @@ -93,7 +93,7 @@ impl Multipart { /// `name` can be either `String` or `&str`, and `read` can take the `Read` by-value or /// with an `&mut` borrow. /// - /// ##Warning + /// ## Warning /// The given `Read` **must** be able to read to EOF (end of file/no more data), meaning /// `Read::read()` returns `Ok(0)`. If it never returns EOF it will be read to infinity /// and the request will never be completed. @@ -104,7 +104,7 @@ impl Multipart { /// Use `Read::take()` if you wish to send data from a `Read` /// that will never return EOF otherwise. /// - /// ##Errors + /// ## Errors /// If the reader returned an error, or if something went wrong with the HTTP stream. // RFC: How to format this declaration? pub fn write_stream, St: Read>( diff --git a/multipart/src/lib.rs b/multipart/src/lib.rs index 53aaa044a..1ed60b52b 100644 --- a/multipart/src/lib.rs +++ b/multipart/src/lib.rs @@ -6,7 +6,7 @@ // copied, modified, or distributed except according to those terms. //! Client- and server-side abstractions for HTTP `multipart/form-data` requests. //! -//! ###Features: +//! ### Features: //! This documentation is built with all features enabled. //! //! * `client`: The client-side abstractions for generating multipart requests. diff --git a/multipart/src/mock.rs b/multipart/src/mock.rs index 538a723dd..f9cb3d336 100644 --- a/multipart/src/mock.rs +++ b/multipart/src/mock.rs @@ -30,7 +30,7 @@ impl ::client::HttpRequest for ClientRequest { true } - /// ##Panics + /// ## Panics /// If `apply_headers()` was not called. fn open_stream(self) -> Result { debug!("ClientRequest::open_stream called! {:?}", self); diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index acaa8e670..8cb12d24f 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -112,7 +112,7 @@ impl Multipart { /// Read the next entry from this multipart request, returning a struct with the field's name and /// data. See `MultipartField` for more info. /// - /// ##Warning: Risk of Data Loss + /// ## Warning: Risk of Data Loss /// If the previously returned entry had contents of type `MultipartField::File`, /// calling this again will discard any unread contents of that entry. pub fn read_entry(&mut self) -> io::Result>> { From 3483c635951ff938041ea7cbcb8fe4a6d2799b3f Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Wed, 14 Jun 2017 21:25:25 -0700 Subject: [PATCH 330/453] Note new minimum Rust version --- multipart/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/README.md b/multipart/README.md index 6970bbbc0..43c037913 100644 --- a/multipart/README.md +++ b/multipart/README.md @@ -4,7 +4,7 @@ Client- and server-side abstractions for HTTP file uploads (POST requests with Supports several different HTTP crates. -Minimum supported Rust version: 1.13.0 (for `?` operator) +Minimum supported Rust version: 1.15.0 ### [Documentation](http://docs.rs/multipart/) From 621a26a169e7dc56990082cf7c59b6c172fa2b2a Mon Sep 17 00:00:00 2001 From: azyobuzin Date: Thu, 22 Jun 2017 20:45:08 +0900 Subject: [PATCH 331/453] Link safemem only if `server` feature is enabled --- multipart/src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/multipart/src/lib.rs b/multipart/src/lib.rs index 1ed60b52b..d23b6fa16 100644 --- a/multipart/src/lib.rs +++ b/multipart/src/lib.rs @@ -48,6 +48,7 @@ extern crate mime_guess; extern crate rand; extern crate tempdir; +#[cfg(feature = "server")] extern crate safemem; #[cfg(feature = "hyper")] From 0d6b2d9ecebab97cb9e4bd80f8c6d1cb88ae8a1b Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Thu, 22 Jun 2017 13:07:07 -0700 Subject: [PATCH 332/453] Bump version to 0.13.1 --- multipart/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index bfd906560..9f2176164 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "multipart" -version = "0.13.0" +version = "0.13.1" authors = ["Austin Bonander "] From e521282d5653ce2d9c7544f673fdab12e4401eea Mon Sep 17 00:00:00 2001 From: Dario Ostuni Date: Sat, 5 Aug 2017 18:44:32 +0200 Subject: [PATCH 333/453] Consume multiple \r\n when needed --- multipart/src/server/boundary.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/multipart/src/server/boundary.rs b/multipart/src/server/boundary.rs index 09533663d..e9b3d8b89 100644 --- a/multipart/src/server/boundary.rs +++ b/multipart/src/server/boundary.rs @@ -131,7 +131,11 @@ impl BoundaryReader where R: Read { let consume_amt = { let buf = self.source.get_buf(); - self.boundary.len() + if buf.len() >=2 && buf[..2] == *b"\r\n" { 2 } else { 0 } + let mut skip_size = 0; + while buf.len() >= skip_size + 2 && buf[skip_size..(skip_size + 2)] == *b"\r\n" { + skip_size += 2; + } + self.boundary.len() + skip_size }; self.source.consume(consume_amt); From e5087f33abf11ef713340b4d073eab8570774cbb Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Wed, 26 Jul 2017 17:31:38 -0700 Subject: [PATCH 334/453] server::Multipart: note about `--` prepended to boundary (cherry picked from commit d88e753) --- multipart/src/server/mod.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index 8cb12d24f..f12030e45 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -99,6 +99,14 @@ impl Multipart<()> { impl Multipart { /// Construct a new `Multipart` with the given body reader and boundary. + /// + /// ## Note: `boundary` + /// This will prepend the requisite `--` to the boundary string as documented in + /// [IETF RFC 1341, Section 7.2.1: "Multipart: the common syntax"][rfc1341-7.2.1]. + /// Simply pass the value of the `boundary` key from the `Content-Type` header in the + /// request (or use `Multipart::from_request()`, if supported). + /// + /// [rfc1341-7.2.1]: https://tools.ietf.org/html/rfc1341#page-30 pub fn with_body>(body: R, boundary: Bnd) -> Self { let boundary = boundary.into(); From ee3a2e5beb7537269ad73649562e05466e558f3e Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Wed, 20 Sep 2017 00:25:09 -0700 Subject: [PATCH 335/453] Modify `iron` and `hyper_client` examples to read out errors --- multipart/examples/hyper_client.rs | 12 ++++++++++-- multipart/examples/iron.rs | 14 ++++++++++++-- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/multipart/examples/hyper_client.rs b/multipart/examples/hyper_client.rs index 0684fecc8..8e968f758 100644 --- a/multipart/examples/hyper_client.rs +++ b/multipart/examples/hyper_client.rs @@ -7,6 +7,8 @@ use hyper::net::Streaming; use multipart::client::Multipart; +use std::io::Read; + fn main() { let url = "http://localhost:80".parse() .expect("Failed to parse URL"); @@ -20,7 +22,13 @@ fn main() { write_body(&mut multipart) .expect("Failed to write multipart body"); - let _response = multipart.send().expect("Failed to send multipart request"); + let mut response = multipart.send().expect("Failed to send multipart request"); + + if !response.status.is_success() { + let mut res = String::new(); + response.read_to_string(&mut res).expect("failed to read response"); + println!("response reported unsuccessful: {:?}\n {}", response, res); + } // Optional: read out response } @@ -33,4 +41,4 @@ fn write_body(multi: &mut Multipart>) -> hyper::Result<()> { // &[u8] impl Read multi.write_stream("binary", &mut binary, None, None) .and(Ok(())) -} \ No newline at end of file +} diff --git a/multipart/examples/iron.rs b/multipart/examples/iron.rs index 4292a2d88..d09d29f28 100644 --- a/multipart/examples/iron.rs +++ b/multipart/examples/iron.rs @@ -1,6 +1,8 @@ extern crate multipart; extern crate iron; +extern crate env_logger; + use std::fs::File; use std::io::Read; use multipart::server::{Multipart, Entries, SaveResult, SavedFile}; @@ -8,6 +10,8 @@ use iron::prelude::*; use iron::status; fn main() { + env_logger::init().unwrap(); + Iron::new(process_request).http("localhost:80").expect("Could not bind localhost:80"); } @@ -23,9 +27,15 @@ fn process_request(request: &mut Request) -> IronResult { SaveResult::Full(entries) => process_entries(entries), SaveResult::Partial(entries, reason) => { process_entries(entries.keep_partial())?; - Err(IronError::new(reason.unwrap_err(), status::InternalServerError)) + Ok(Response::with(( + status::BadRequest, + format!("error reading request: {}", reason.unwrap_err()) + ))) } - SaveResult::Error(error) => Err(IronError::new(error, status::InternalServerError)), + SaveResult::Error(error) => Ok(Response::with(( + status::BadRequest, + format!("error reading request: {}", error) + ))), } } Err(_) => { From efaea40429a4c2891e25dcd8adb523ecdcc5bf70 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Wed, 20 Sep 2017 00:26:20 -0700 Subject: [PATCH 336/453] Minor bug fixes --- multipart/src/server/boundary.rs | 65 +++++++++++++++++++++++--------- multipart/src/server/field.rs | 2 +- 2 files changed, 48 insertions(+), 19 deletions(-) diff --git a/multipart/src/server/boundary.rs b/multipart/src/server/boundary.rs index e9b3d8b89..354d4262c 100644 --- a/multipart/src/server/boundary.rs +++ b/multipart/src/server/boundary.rs @@ -130,30 +130,33 @@ impl BoundaryReader where R: Read { String::from_utf8_lossy(self.source.get_buf())); let consume_amt = { - let buf = self.source.get_buf(); - let mut skip_size = 0; - while buf.len() >= skip_size + 2 && buf[skip_size..(skip_size + 2)] == *b"\r\n" { - skip_size += 2; + let min_len = self.boundary.len() + 4; + + let buf = fill_buf_min(&mut self.source, min_len)?; + + if buf.len() < min_len { + return Err(io::Error::new(io::ErrorKind::UnexpectedEof, + "not enough bytes to verify boundary")); } - self.boundary.len() + skip_size - }; - self.source.consume(consume_amt); - self.boundary_read = false; + let mut consume_amt = self.boundary.len(); - let mut bytes_after = [0, 0]; + if buf[..2] == *b"\r\n" { consume_amt += 2 } - let read = self.source.read(&mut bytes_after)?; + let last_two = &buf[consume_amt .. consume_amt + 2]; - if read == 1 { - let _ = self.source.read(&mut bytes_after[1..])?; - } + match last_two { + b"\r\n" => consume_amt += 2, + b"--" => { consume_amt += 2; self.at_end = true }, + _ => debug!("Unexpected bytes following boundary: {:?}", + String::from_utf8_lossy(&last_two)), + } - if bytes_after == *b"--" { - self.at_end = true; - } else if bytes_after != *b"\r\n" { - debug!("Unexpected bytes following boundary: {:?}", String::from_utf8_lossy(&bytes_after)); - } + consume_amt + }; + + self.source.consume(consume_amt); + self.boundary_read = false; trace!("Consumed boundary (at_end: {:?}), remaining buf: {:?}", self.at_end, String::from_utf8_lossy(self.source.get_buf())); @@ -333,6 +336,32 @@ mod test { assert_eq!(buf, ""); } + #[test] + fn test_leading_crlf() { + let mut body: &[u8] = b"\r\n\r\n--boundary\r\n\ + asdf1234\ + \r\n\r\n--boundary--"; + + let ref mut buf = String::new(); + let mut reader = BoundaryReader::from_reader(&mut body, BOUNDARY); + + + debug!("Consume 1"); + reader.consume_boundary().unwrap(); + + debug!("Read 1"); + let _ = reader.read_to_string(buf).unwrap(); + assert_eq!(buf, "asdf1234\r\n"); + buf.clear(); + + debug!("Consume 2"); + reader.consume_boundary().unwrap(); + + debug!("Read 2 (empty)"); + let _ = reader.read_to_string(buf).unwrap(); + assert_eq!(buf, ""); + } + #[cfg(feature = "bench")] mod bench { extern crate test; diff --git a/multipart/src/server/field.rs b/multipart/src/server/field.rs index d1f82048b..8e2e90c85 100644 --- a/multipart/src/server/field.rs +++ b/multipart/src/server/field.rs @@ -32,7 +32,7 @@ pub struct StrHeader<'a> { val: &'a str, } -const MAX_ATTEMPTS: usize = 5; +const MAX_ATTEMPTS: usize = 30; fn with_headers(r: &mut R, closure: F) -> Result where R: BufRead, F: FnOnce(&[StrHeader]) -> Ret { From a7d60a94eb8cc50f0606b0cabf01de970ae4b6f2 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Wed, 20 Sep 2017 12:35:13 -0700 Subject: [PATCH 337/453] Bump minimum supported Rust version --- multipart/.travis.yml | 2 +- multipart/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/multipart/.travis.yml b/multipart/.travis.yml index ad7776428..0728306ed 100644 --- a/multipart/.travis.yml +++ b/multipart/.travis.yml @@ -5,7 +5,7 @@ branches: except: - fuzzing rust: - - 1.15.0 + - 1.16.0 - stable - beta - nightly diff --git a/multipart/README.md b/multipart/README.md index 43c037913..70f515ba3 100644 --- a/multipart/README.md +++ b/multipart/README.md @@ -4,7 +4,7 @@ Client- and server-side abstractions for HTTP file uploads (POST requests with Supports several different HTTP crates. -Minimum supported Rust version: 1.15.0 +Minimum supported Rust version: 1.16.0 ### [Documentation](http://docs.rs/multipart/) From 9d9c77862816a4fa3f33b7500a0d4833dcbdea71 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Wed, 20 Sep 2017 12:35:41 -0700 Subject: [PATCH 338/453] Bump version --- multipart/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 9f2176164..64c0d1e06 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "multipart" -version = "0.13.1" +version = "0.13.2" authors = ["Austin Bonander "] From 97262c4faff988340cb51f31ad8ca8c22be188b5 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Wed, 20 Sep 2017 12:37:09 -0700 Subject: [PATCH 339/453] Move `lorem_ipsum.txt` to root --- multipart/{examples => }/lorem_ipsum.txt | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename multipart/{examples => }/lorem_ipsum.txt (100%) diff --git a/multipart/examples/lorem_ipsum.txt b/multipart/lorem_ipsum.txt similarity index 100% rename from multipart/examples/lorem_ipsum.txt rename to multipart/lorem_ipsum.txt From 595219e3b82d49987c16d69872519a1cbbb837a3 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Wed, 20 Sep 2017 12:45:24 -0700 Subject: [PATCH 340/453] Minimum Rust version to 1.17.0 --- multipart/.travis.yml | 2 +- multipart/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/multipart/.travis.yml b/multipart/.travis.yml index 0728306ed..86def6adc 100644 --- a/multipart/.travis.yml +++ b/multipart/.travis.yml @@ -5,7 +5,7 @@ branches: except: - fuzzing rust: - - 1.16.0 + - 1.17.0 - stable - beta - nightly diff --git a/multipart/README.md b/multipart/README.md index 70f515ba3..67e9c8fa1 100644 --- a/multipart/README.md +++ b/multipart/README.md @@ -4,7 +4,7 @@ Client- and server-side abstractions for HTTP file uploads (POST requests with Supports several different HTTP crates. -Minimum supported Rust version: 1.16.0 +Minimum supported Rust version: 1.17.0 ### [Documentation](http://docs.rs/multipart/) From d42257c8498e983a987e09a538f52fe4883ad1de Mon Sep 17 00:00:00 2001 From: little-bobby-tables Date: Fri, 22 Sep 2017 12:33:08 +0300 Subject: [PATCH 341/453] compare header names case-insentively (rfc 822) --- multipart/src/server/field.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/multipart/src/server/field.rs b/multipart/src/server/field.rs index 8e2e90c85..1b6506026 100644 --- a/multipart/src/server/field.rs +++ b/multipart/src/server/field.rs @@ -20,6 +20,8 @@ use std::ops::Deref; use std::path::{Path, PathBuf}; use std::{str, fmt, error}; +use std::ascii::AsciiExt; + const EMPTY_STR_HEADER: StrHeader<'static> = StrHeader { name: "", val: "", @@ -516,7 +518,9 @@ fn io_str_utf8(buf: &[u8]) -> io::Result<&str> { } fn find_header<'a, 'b>(headers: &'a [StrHeader<'b>], name: &str) -> Option<&'a StrHeader<'b>> { - headers.iter().find(|header| header.name == name) + /// Field names are case insensitive and consist of ASCII characters + /// only (see https://tools.ietf.org/html/rfc822#section-3.2). + headers.iter().find(|header| header.name.eq_ignore_ascii_case(name)) } /// Common trait for `Multipart` and `&mut Multipart` From fea274058f2aa05696088ae7f7ca94dd5aae5fee Mon Sep 17 00:00:00 2001 From: little-bobby-tables Date: Sat, 23 Sep 2017 08:29:02 +0300 Subject: [PATCH 342/453] test header name comparison --- multipart/src/server/field.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/multipart/src/server/field.rs b/multipart/src/server/field.rs index 1b6506026..5918ef725 100644 --- a/multipart/src/server/field.rs +++ b/multipart/src/server/field.rs @@ -743,3 +743,16 @@ impl From for ParseHeaderError { ParseHeaderError::Invalid(format!("{}", err)) } } + +#[test] +fn test_find_header() { + let headers = [ + StrHeader { name: "Content-Type", val: "text/plain" }, + StrHeader { name: "Content-disposition", val: "form-data" }, + StrHeader { name: "content-transfer-encoding", val: "binary" } + ]; + + assert_eq!(find_header(&headers, "Content-Type").unwrap().val, "text/plain"); + assert_eq!(find_header(&headers, "Content-Disposition").unwrap().val, "form-data"); + assert_eq!(find_header(&headers, "Content-Transfer-Encoding").unwrap().val, "binary"); +} From 364b043057eac9fa1e6a563807497c308fd3ffa6 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Fri, 22 Sep 2017 23:07:23 -0700 Subject: [PATCH 343/453] Bump version --- multipart/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 64c0d1e06..35c7c1df1 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "multipart" -version = "0.13.2" +version = "0.13.3" authors = ["Austin Bonander "] From cd756055600379aa7dd93be351441f7e3c1618a9 Mon Sep 17 00:00:00 2001 From: Jacob Brown Date: Thu, 19 Oct 2017 02:43:16 -0500 Subject: [PATCH 344/453] change fields container for Entries (#91) Change fields container for Entries * don't need drain here * add IntoIterator for FieldsWrapper, cleanup examples * add From impl --- multipart/src/server/save.rs | 59 ++++++++++++++++++++++++++++++++++-- 1 file changed, 56 insertions(+), 3 deletions(-) diff --git a/multipart/src/server/save.rs b/multipart/src/server/save.rs index 3b7666151..227c84de6 100644 --- a/multipart/src/server/save.rs +++ b/multipart/src/server/save.rs @@ -231,7 +231,7 @@ impl SaveBuilder where M: ReadEntry { }, MultipartData::Text(mut text) => { self.savable = text.take_inner(); - entries.fields.insert(field.name, text.text); + entries.fields.push((field.name, text.text)); }, } } @@ -388,11 +388,64 @@ impl SavedFile { } } +/// Wrapper for the fields of an `Entries`. Stores text fields in a `Vec<(String, String)>` to +/// enable accessing fields with duplicated names. + +#[derive(Debug, Default)] +pub struct FieldsWrapper(Vec<(String, String)>); + +impl FieldsWrapper { + + /// Converts the contents into a multivalued hashmap + pub fn into_multivalued_map(self) -> HashMap> { + let mut out = HashMap::new(); + for (key, val) in self.0.into_iter() { + out.entry(key).or_insert_with(Vec::new).push(val); + } + out + } + + /// Converts the contents into a single-valued hashmap + pub fn into_map(self) -> HashMap { + self.0.into_iter().collect() + } +} + +impl IntoIterator for FieldsWrapper { + type Item = (String, String); + type IntoIter = ::std::vec::IntoIter; + + fn into_iter(self) -> Self::IntoIter { + self.0.into_iter() + } +} + +impl ::std::ops::Deref for FieldsWrapper { + type Target = Vec<(String, String)>; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl ::std::ops::DerefMut for FieldsWrapper { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +// this allows use of both `from(FieldsWrapper)` and `FieldsWrapper.into()` +impl From for Vec<(String, String)> { + fn from(fields: FieldsWrapper) -> Self { + fields.0 + } +} + /// A result of `Multipart::save_all()`. #[derive(Debug)] pub struct Entries { /// The text fields of the multipart request, mapped by field name -> value. - pub fields: HashMap, + pub fields: FieldsWrapper, /// A map of file field names to their contents saved on the filesystem. pub files: HashMap>, /// The directory the files in this request were saved under; may be temporary or permanent. @@ -402,7 +455,7 @@ pub struct Entries { impl Entries { fn new(save_dir: SaveDir) -> Self { Entries { - fields: HashMap::new(), + fields: FieldsWrapper::default(), files: HashMap::new(), save_dir: save_dir, } From 9bc93653f66e0f4d89728015efd6c64ce8c95d2b Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sun, 26 Nov 2017 11:26:13 -0800 Subject: [PATCH 345/453] Add test for a field with a trailing CRLF --- multipart/src/server/boundary.rs | 34 ++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/multipart/src/server/boundary.rs b/multipart/src/server/boundary.rs index 354d4262c..04ab12004 100644 --- a/multipart/src/server/boundary.rs +++ b/multipart/src/server/boundary.rs @@ -362,6 +362,40 @@ mod test { assert_eq!(buf, ""); } + #[test] + fn test_trailing_crlf() { + let mut body: &[u8] = b"--boundary\r\n\ + asdf1234\ + \r\n\r\n--boundary\r\n\ + hjkl5678\r\n--boundary--"; + + let ref mut buf = String::new(); + let mut reader = BoundaryReader::from_reader(&mut body, BOUNDARY); + + debug!("Consume 1"); + reader.consume_boundary().unwrap(); + + debug!("Read 1"); + let _ = reader.read_to_string(buf).unwrap(); + assert_eq!(buf, "asdf1234\r\n"); + buf.clear(); + + debug!("Consume 2"); + reader.consume_boundary().unwrap(); + + debug!("Read 2"); + let _ = reader.read_to_string(buf).unwrap(); + assert_eq!(buf, "hjkl5678"); + buf.clear(); + + debug!("Consume 3"); + reader.consume_boundary().unwrap(); + + debug!("Read 3 (empty)"); + let _ = reader.read_to_string(buf).unwrap(); + assert_eq!(buf, ""); + } + #[cfg(feature = "bench")] mod bench { extern crate test; From 24c40b8613bc6284a2ecef0a56d35403c6390c43 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sun, 26 Nov 2017 14:46:18 -0800 Subject: [PATCH 346/453] clarify `BoundaryReader` possible states as enum, fix cursor backup for preceding CRLF --- multipart/src/server/boundary.rs | 132 +++++++++++++++++++++---------- 1 file changed, 90 insertions(+), 42 deletions(-) diff --git a/multipart/src/server/boundary.rs b/multipart/src/server/boundary.rs index 04ab12004..8c1d064ea 100644 --- a/multipart/src/server/boundary.rs +++ b/multipart/src/server/boundary.rs @@ -20,14 +20,22 @@ use std::borrow::Borrow; use std::io; use std::io::prelude::*; +use self::State::*; + +#[derive(Debug, PartialEq, Eq)] +enum State { + Searching, + BoundaryRead, + AtEnd +} + /// A struct implementing `Read` and `BufRead` that will yield bytes until it sees a given sequence. #[derive(Debug)] pub struct BoundaryReader { source: BufReader, boundary: Vec, search_idx: usize, - boundary_read: bool, - at_end: bool, + state: State, } impl BoundaryReader where R: Read { @@ -40,8 +48,7 @@ impl BoundaryReader where R: Read { source: BufReader::new(reader), boundary: boundary, search_idx: 0, - boundary_read: false, - at_end: false, + state: Searching, } } @@ -53,16 +60,14 @@ impl BoundaryReader where R: Read { if buf.is_empty() { debug!("fill_buf_min returned zero-sized buf"); - self.at_end = true; - return Ok(buf); } trace!("Buf: {:?}", String::from_utf8_lossy(buf)); - debug!("Before-loop Buf len: {} Search idx: {} Boundary read: {:?}", - buf.len(), self.search_idx, self.boundary_read); + debug!("Before-loop Buf len: {} Search idx: {} State: {:?}", + buf.len(), self.search_idx, self.state); - if !self.boundary_read && self.search_idx < buf.len() { + if self.state == Searching && self.search_idx < buf.len() { let lookahead = &buf[self.search_idx..]; debug!("Find boundary loop! Lookahead len: {}", lookahead.len()); @@ -71,7 +76,7 @@ impl BoundaryReader where R: Read { match twoway::find_bytes(lookahead, &self.boundary) { Some(found_idx) => { self.search_idx += found_idx; - self.boundary_read = true; + self.state = BoundaryRead; }, None => { self.search_idx += lookahead.len().saturating_sub(self.boundary.len() + 2); @@ -79,24 +84,26 @@ impl BoundaryReader where R: Read { } } - debug!("After-loop Buf len: {} Search idx: {} Boundary read: {:?}", - buf.len(), self.search_idx, self.boundary_read); + debug!("After-loop Buf len: {} Search idx: {} State: {:?}", + buf.len(), self.search_idx, self.state); + + // don't modify search_idx so it always points to the start of the boundary + let mut buf_len = self.search_idx; - // If the two bytes before the boundary are a CR-LF, we need to back up - // the cursor so we don't yield bytes that client code isn't expecting. - if self.boundary_read && self.search_idx >= 2 { - let two_bytes_before = &buf[self.search_idx - 2 .. self.search_idx]; + // back up the cursor to before the boundary's preceding CRLF + if self.state != Searching && buf_len >= 2 { + let two_bytes_before = &buf[buf_len - 2 .. buf_len]; trace!("Two bytes before: {:?} ({:?}) (\"\\r\\n\": {:?})", String::from_utf8_lossy(two_bytes_before), two_bytes_before, b"\r\n"); if two_bytes_before == &*b"\r\n" { debug!("Subtract two!"); - self.search_idx -= 2; - } + buf_len -= 2; + } } - let ret_buf = &buf[..self.search_idx]; + let ret_buf = &buf[..buf_len]; trace!("Returning buf: {:?}", String::from_utf8_lossy(ret_buf)); @@ -105,30 +112,20 @@ impl BoundaryReader where R: Read { #[doc(hidden)] pub fn consume_boundary(&mut self) -> io::Result { - if self.at_end { + if self.state == AtEnd { return Ok(true); } - while !(self.boundary_read || self.at_end){ + while self.state == Searching { debug!("Boundary not found yet"); let buf_len = self.read_to_boundary()?.len(); debug!("Discarding {} bytes", buf_len); - if buf_len == 0 { - break; - } - self.consume(buf_len); } - self.source.consume(self.search_idx); - self.search_idx = 0; - - trace!("Consumed up to self.search_idx, remaining buf: {:?}", - String::from_utf8_lossy(self.source.get_buf())); - let consume_amt = { let min_len = self.boundary.len() + 4; @@ -139,15 +136,16 @@ impl BoundaryReader where R: Read { "not enough bytes to verify boundary")); } - let mut consume_amt = self.boundary.len(); + // we have enough bytes to verify + self.state = Searching; - if buf[..2] == *b"\r\n" { consume_amt += 2 } + let mut consume_amt = self.search_idx + self.boundary.len(); let last_two = &buf[consume_amt .. consume_amt + 2]; match last_two { b"\r\n" => consume_amt += 2, - b"--" => { consume_amt += 2; self.at_end = true }, + b"--" => { consume_amt += 2; self.state = AtEnd }, _ => debug!("Unexpected bytes following boundary: {:?}", String::from_utf8_lossy(&last_two)), } @@ -155,13 +153,17 @@ impl BoundaryReader where R: Read { consume_amt }; + trace!("Consuming {} bytes, remaining buf: {:?}", + consume_amt, + String::from_utf8_lossy(self.source.get_buf())); + self.source.consume(consume_amt); - self.boundary_read = false; + self.search_idx = 0; - trace!("Consumed boundary (at_end: {:?}), remaining buf: {:?}", self.at_end, + trace!("Consumed boundary (state: {:?}), remaining buf: {:?}", self.state, String::from_utf8_lossy(self.source.get_buf())); - Ok(self.at_end) + Ok(self.state == AtEnd) } } @@ -174,8 +176,7 @@ impl<'a> BoundaryReader> { fn reset(&mut self) { // Dump buffer and reset cursor self.source.seek(io::SeekFrom::Start(0)); - self.at_end = false; - self.boundary_read = false; + self.state = Searching; self.search_idx = 0; } } @@ -215,11 +216,9 @@ impl BufRead for BoundaryReader where R: Read { } fn fill_buf_min(buf: &mut BufReader, min: usize) -> io::Result<&[u8]> { - const MAX_ATTEMPTS: usize = 3; - let mut attempts = 0; - while buf.available() < min && attempts < MAX_ATTEMPTS { + while buf.available() < min && attempts < min { if buf.read_into_buf()? == 0 { break; }; attempts += 1; } @@ -233,6 +232,7 @@ mod test { use std::io; use std::io::prelude::*; + use std::slice; const BOUNDARY: &'static str = "boundary"; const TEST_VAL: &'static str = "--boundary\r\n\ @@ -376,6 +376,13 @@ mod test { reader.consume_boundary().unwrap(); debug!("Read 1"); + + // Repro for https://github.com/abonander/multipart/issues/93 + // These two reads should produce the same buffer + let buf1 = reader.read_to_boundary().unwrap().to_owned(); + let buf2 = reader.read_to_boundary().unwrap().to_owned(); + assert_eq!(buf1, buf2); + let _ = reader.read_to_string(buf).unwrap(); assert_eq!(buf, "asdf1234\r\n"); buf.clear(); @@ -396,6 +403,47 @@ mod test { assert_eq!(buf, ""); } + // https://github.com/abonander/multipart/issues/93#issuecomment-343610587 + #[test] + fn test_trailing_lflf() { + let mut body: &[u8] = b"--boundary\r\n\ + asdf1234\ + \n\n\r\n--boundary\r\n\ + hjkl5678\r\n--boundary--"; + + let ref mut buf = String::new(); + let mut reader = BoundaryReader::from_reader(&mut body, BOUNDARY); + + debug!("Consume 1"); + reader.consume_boundary().unwrap(); + + debug!("Read 1"); + + // same as above + let buf1 = reader.read_to_boundary().unwrap().to_owned(); + let buf2 = reader.read_to_boundary().unwrap().to_owned(); + assert_eq!(buf1, buf2); + + let _ = reader.read_to_string(buf).unwrap(); + assert_eq!(buf, "asdf1234\n\n"); + buf.clear(); + + debug!("Consume 2"); + reader.consume_boundary().unwrap(); + + debug!("Read 2"); + let _ = reader.read_to_string(buf).unwrap(); + assert_eq!(buf, "hjkl5678"); + buf.clear(); + + debug!("Consume 3"); + reader.consume_boundary().unwrap(); + + debug!("Read 3 (empty)"); + let _ = reader.read_to_string(buf).unwrap(); + assert_eq!(buf, ""); + } + #[cfg(feature = "bench")] mod bench { extern crate test; From 07c359d5c1cf23a5aeb4339b6c9cd13925ec0097 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sun, 26 Nov 2017 16:07:47 -0800 Subject: [PATCH 347/453] Bump version for `BoundaryReader` fix --- multipart/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 35c7c1df1..95e8e329d 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "multipart" -version = "0.13.3" +version = "0.13.4" authors = ["Austin Bonander "] From 0438e20aa42b445c996a2a6429b8cddd827f485f Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Wed, 26 Jul 2017 16:16:36 -0700 Subject: [PATCH 348/453] server::Multipart: remove deprecated methods --- multipart/src/server/mod.rs | 50 ------------------------------------- 1 file changed, 50 deletions(-) diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index f12030e45..ffafe040a 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -155,56 +155,6 @@ impl Multipart { pub fn save(&mut self) -> SaveBuilder<&mut Self> { SaveBuilder::new(self) } - - /// Read the request fully, parsing all fields and saving all files in a new temporary - /// directory under the OS temporary directory. - /// - /// If there is an error in reading the request, returns the partial result along with the - /// error. See [`SaveResult`](save/enum.SaveResult.html) for more information. - #[deprecated(since = "0.10.0", note = "use `.save().temp()` instead")] - pub fn save_all(&mut self) -> EntriesSaveResult<&mut Self> { - self.save().temp() - } - - /// Read the request fully, parsing all fields and saving all files in a new temporary - /// directory under `dir`. - /// - /// If there is an error in reading the request, returns the partial result along with the - /// error. See [`SaveResult`](save/enum.SaveResult.html) for more information. - #[deprecated(since = "0.10.0", note = "use `.save().with_temp_dir()` instead")] - pub fn save_all_under>(&mut self, dir: P) -> EntriesSaveResult<&mut Self> { - match TempDir::new_in(dir, "multipart") { - Ok(temp_dir) => self.save().with_temp_dir(temp_dir), - Err(err) => SaveResult::Error(err), - } - } - - /// Read the request fully, parsing all fields and saving all fields in a new temporary - /// directory under the OS temporary directory. - /// - /// Files larger than `limit` will be truncated to `limit`. - /// - /// If there is an error in reading the request, returns the partial result along with the - /// error. See [`SaveResult`](save/enum.SaveResult.html) for more information. - #[deprecated(since = "0.10.0", note = "use `.save().size_limit(limit)` instead")] - pub fn save_all_limited(&mut self, limit: u64) -> EntriesSaveResult<&mut Self> { - self.save().size_limit(limit).temp() - } - - /// Read the request fully, parsing all fields and saving all files in a new temporary - /// directory under `dir`. - /// - /// Files larger than `limit` will be truncated to `limit`. - /// - /// If there is an error in reading the request, returns the partial result along with the - /// error. See [`SaveResult`](save/enum.SaveResult.html) for more information. - #[deprecated(since = "0.10.0", note = "use `.save().size_limit(limit).with_temp_dir()` instead")] - pub fn save_all_under_limited>(&mut self, dir: P, limit: u64) -> EntriesSaveResult<&mut Self> { - match TempDir::new_in(dir, "multipart") { - Ok(temp_dir) => self.save().size_limit(limit).with_temp_dir(temp_dir), - Err(err) => SaveResult::Error(err), - } - } } impl Borrow for Multipart { From 3b2e66a5a1a556fcfb5d635f554866bac854342d Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sun, 26 Nov 2017 22:40:12 -0800 Subject: [PATCH 349/453] WIP removing text/data distinction in API --- multipart/src/server/field.rs | 279 ++++------------------------------ 1 file changed, 29 insertions(+), 250 deletions(-) diff --git a/multipart/src/server/field.rs b/multipart/src/server/field.rs index 5918ef725..0cc45ff1f 100644 --- a/multipart/src/server/field.rs +++ b/multipart/src/server/field.rs @@ -172,12 +172,29 @@ fn parse_cont_type(headers: &[StrHeader]) -> Result, ParseHeaderErr Ok(Some(read_content_type(header.val.trim()))) } -/// A field in a multipart request. May be either text or a binary stream (file). +/// A field in a multipart request with its associated headers and data. +/// +/// ### Warning: Values are Client-Provided +/// Everything in this struct are values from the client and should be considered **untrustworthy**. +/// This crate makes no effort to validate or sanitize any client inputs. #[derive(Debug)] pub struct MultipartField { - /// The field's name from the form + /// The field's name from the form. pub name: String, - /// The data of the field. Can be text or binary. + + /// The filename of this entry, if supplied. This is not guaranteed to match the original file + /// or even to be a valid filename for the current platform. + pub filename: Option, + + /// The MIME type (`Content-Type` value) of this file, if supplied by the client. + /// + /// If this is not supplied, the content-type of the field should default to `text/plain` as + /// per [IETF RFC 7578, section 4.4](https://tools.ietf.org/html/rfc7578#section-4.4), but this + /// should not be implicitly trusted. This crate makes no attempt to identify or validate + /// the content-type of the actual field data. + pub content_type: Option, + + /// The field's data. pub data: MultipartData, } @@ -212,185 +229,25 @@ impl MultipartField { } /// The data of a field in a `multipart/form-data` request. -#[derive(Debug)] -pub enum MultipartData { - /// The field's payload is a text string. - Text(MultipartText), - /// The field's payload is a binary stream (file). - File(MultipartFile), -} - -impl MultipartData { - /// Borrow this payload as a text field, if possible. - pub fn as_text(&self) -> Option<&str> { - match *self { - MultipartData::Text(ref text) => Some(&text.text), - _ => None, - } - } - - /// Borrow this payload as a file field, if possible. - /// Mutably borrows so the contents can be read. - pub fn as_file(&mut self) -> Option<&mut MultipartFile> { - match *self { - MultipartData::File(ref mut file) => Some(file), - _ => None, - } - } - - /// Return the inner `Multipart`. - pub fn into_inner(self) -> M { - use self::MultipartData::*; - - match self { - Text(text) => text.into_inner(), - File(file) => file.into_inner(), - } - } - - fn take_inner(&mut self) -> M { - use self::MultipartData::*; - - match *self { - Text(ref mut text) => text.take_inner(), - File(ref mut file) => file.take_inner(), - } - } - - fn give_inner(&mut self, inner: M) { - use self::MultipartData::*; - - let inner = Some(inner); - - match *self { - Text(ref mut text) => text.inner = inner, - File(ref mut file) => file.inner = inner, - } - } -} - -/// A representation of a text field in a `multipart/form-data` body. -#[derive(Debug)] -pub struct MultipartText { - /// The text of this field. - pub text: String, - /// The `Multipart` this field was read from. - inner: Option, -} - -impl Deref for MultipartText { - type Target = str; - - fn deref(&self) -> &Self::Target { - &self.text - } -} - -impl Into for MultipartText { - fn into(self) -> String { - self.text - } -} - -impl MultipartText { - #[doc(hidden)] - pub fn take_inner(&mut self) -> M { - self.inner.take().expect("MultipartText::inner already taken!") - } - - fn into_inner(self) -> M { - self.inner.expect("MultipartText::inner taken!") - } -} - -/// A representation of a file in HTTP `multipart/form-data`. -/// -/// Note that the file is not yet saved to the local filesystem; -/// instead, this struct exposes `Read` and `BufRead` impls which point -/// to the beginning of the file's contents in the HTTP stream. /// /// You can read it to EOF, or use one of the `save()` method -/// to save it to disk. +/// to save it to disk/memory. #[derive(Debug)] -pub struct MultipartFile { - /// The filename of this entry, if supplied. - /// - /// ### Warning: Client Provided / Untrustworthy - /// You should treat this value as **untrustworthy** because it is an arbitrary string - /// provided by the client. - /// - /// It is a serious security risk to create files or directories with paths based on user input. - /// A malicious user could craft a path which can be used to overwrite important files, such as - /// web templates, static assets, Javascript files, database files, configuration files, etc., - /// if they are writable by the server process. - /// - /// This can be mitigated somewhat by setting filesystem permissions as - /// conservatively as possible and running the server under its own user with restricted - /// permissions, but you should still not use user input directly as filesystem paths. - /// If it is truly necessary, you should sanitize filenames such that they cannot be - /// misinterpreted by the OS. Such functionality is outside the scope of this crate. - pub filename: Option, - - /// The MIME type (`Content-Type` value) of this file, if supplied by the client, - /// or `"applicaton/octet-stream"` otherwise. - /// - /// ### Note: Client Provided - /// Consider this value to be potentially untrustworthy, as it is provided by the client. - /// It may be inaccurate or entirely wrong, depending on how the client determined it. - /// - /// Some variants wrap arbitrary strings which could be abused by a malicious user if your - /// application performs any non-idempotent operations based on their value, such as - /// starting another program or querying/updating a database (web-search "SQL injection"). - pub content_type: Mime, - - /// The `Multipart` this field was read from. +pub struct MultipartData { inner: Option, } -impl MultipartFile { - /// Get the filename of this entry, if supplied. - /// - /// ### Warning: Client Provided / Untrustworthy - /// You should treat this value as **untrustworthy** because it is an arbitrary string - /// provided by the client. - /// - /// It is a serious security risk to create files or directories with paths based on user input. - /// A malicious user could craft a path which can be used to overwrite important files, such as - /// web templates, static assets, Javascript files, database files, configuration files, etc., - /// if they are writable by the server process. - /// - /// This can be mitigated somewhat by setting filesystem permissions as - /// conservatively as possible and running the server under its own user with restricted - /// permissions, but you should still not use user input directly as filesystem paths. - /// If it is truly necessary, you should sanitize filenames such that they cannot be - /// misinterpreted by the OS. Such functionality is outside the scope of this crate. - #[deprecated(since = "0.10.0", note = "`filename` field is now public")] - pub fn filename(&self) -> Option<&str> { - self.filename.as_ref().map(String::as_ref) - } - - /// Get the MIME type (`Content-Type` value) of this file, if supplied by the client, - /// or `"applicaton/octet-stream"` otherwise. - /// - /// ### Note: Client Provided - /// Consider this value to be potentially untrustworthy, as it is provided by the client. - /// It may be inaccurate or entirely wrong, depending on how the client determined it. - /// - /// Some variants wrap arbitrary strings which could be abused by a malicious user if your - /// application performs any non-idempotent operations based on their value, such as - /// starting another program or querying/updating a database (web-search "SQL injection"). - #[deprecated(since = "0.10.0", note = "`content_type` field is now public")] - pub fn content_type(&self) -> &Mime { - &self.content_type +impl MultipartData where M: ReadEntry { + /// Get a builder type which can save the file with or without a size limit. + pub fn save(&mut self) -> SaveBuilder<&mut Self> { + SaveBuilder::new(self) } - fn inner_mut(&mut self) -> &mut M { self.inner.as_mut().expect("MultipartFile::inner taken!") } - #[doc(hidden)] - pub fn take_inner(&mut self) -> M { + fn take_inner(&mut self) -> M { self.inner.take().expect("MultipartFile::inner already taken!") } @@ -399,91 +256,13 @@ impl MultipartFile { } } -impl MultipartFile where M: ReadEntry { - /// Get a builder type which can save the file with or without a size limit. - pub fn save(&mut self) -> SaveBuilder<&mut Self> { - SaveBuilder::new(self) - } - - /// Save this file to the given output stream. - /// - /// If successful, returns the number of bytes written. - /// - /// Retries when `io::Error::kind() == io::ErrorKind::Interrupted`. - #[deprecated(since = "0.10.0", note = "use `.save().write_to()` instead")] - pub fn save_to(&mut self, out: W) -> io::Result { - self.save().write_to(out).into_result_strict() - } - - /// Save this file to the given output stream, **truncated** to `limit` - /// (no more than `limit` bytes will be written out). - /// - /// If successful, returns the number of bytes written. - /// - /// Retries when `io::Error::kind() == io::ErrorKind::Interrupted`. - #[deprecated(since = "0.10.0", note = "use `.save().size_limit(limit).write_to(out)` instead")] - pub fn save_to_limited(&mut self, out: W, limit: u64) -> io::Result { - self.save().size_limit(limit).write_to(out).into_result_strict() - } - - /// Save this file to `path`. - /// - /// Returns the saved file info on success, or any errors otherwise. - /// - /// Retries when `io::Error::kind() == io::ErrorKind::Interrupted`. - #[deprecated(since = "0.10.0", note = "use `.save().with_path(path)` instead")] - pub fn save_as>(&mut self, path: P) -> io::Result { - self.save().with_path(path).into_result_strict() - } - - /// Save this file in the directory pointed at by `dir`, - /// using a random alphanumeric string as the filename. - /// - /// Any missing directories in the `dir` path will be created. - /// - /// Returns the saved file's info on success, or any errors otherwise. - /// - /// Retries when `io::Error::kind() == io::ErrorKind::Interrupted`. - #[deprecated(since = "0.10.0", note = "use `.save().with_dir(dir)` instead")] - pub fn save_in>(&mut self, dir: P) -> io::Result { - self.save().with_dir(dir.as_ref()).into_result_strict() - } - - /// Save this file to `path`, **truncated** to `limit` (no more than `limit` bytes will be written out). - /// - /// Any missing directories in the `dir` path will be created. - /// - /// Returns the saved file's info on success, or any errors otherwise. - /// - /// Retries when `io::Error::kind() == io::ErrorKind::Interrupted`. - #[deprecated(since = "0.10.0", note = "use `.save().size_limit(limit).with_path(path)` instead")] - pub fn save_as_limited>(&mut self, path: P, limit: u64) -> io::Result { - self.save().size_limit(limit).with_path(path).into_result_strict() - } - - /// Save this file in the directory pointed at by `dir`, - /// using a random alphanumeric string as the filename. - /// - /// **Truncates** file to `limit` (no more than `limit` bytes will be written out). - /// - /// Any missing directories in the `dir` path will be created. - /// - /// Returns the saved file's info on success, or any errors otherwise. - /// - /// Retries when `io::Error::kind() == io::ErrorKind::Interrupted`. - #[deprecated(since = "0.10.0", note = "use `.save().size_limit(limit).with_dir(dir)` instead")] - pub fn save_in_limited>(&mut self, dir: P, limit: u64) -> io::Result { - self.save().size_limit(limit).with_dir(dir).into_result_strict() - } -} - -impl Read for MultipartFile { +impl Read for MultipartData { fn read(&mut self, buf: &mut [u8]) -> io::Result{ self.inner_mut().source().read(buf) } } -impl BufRead for MultipartFile { +impl BufRead for MultipartData { fn fill_buf(&mut self) -> io::Result<&[u8]> { self.inner_mut().source().fill_buf() } From c1a3d2fbc061e532942fb329e5782e40bec04521 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Mon, 4 Dec 2017 03:35:02 -0800 Subject: [PATCH 350/453] WIP refactoring --- multipart/Cargo.toml | 9 ++- multipart/src/server/field.rs | 117 +++++++++++++++------------- multipart/src/server/save.rs | 142 +++++----------------------------- 3 files changed, 85 insertions(+), 183 deletions(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 95e8e329d..cd1d4506f 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "multipart" -version = "0.13.4" +version = "0.14.0" authors = ["Austin Bonander "] @@ -17,8 +17,8 @@ license = "MIT OR Apache-2.0" [dependencies] log = "0.3" -mime = "0.2" -mime_guess = "1.8" +mime = "0.3" +mime_guess = "2.0.0-alpha.3" rand = "0.3" safemem = { version = "0.2", optional = true } tempdir = ">=0.3.4" @@ -27,6 +27,7 @@ clippy = { version = ">=0.0, <0.1", optional = true} #Server Dependencies buf_redux = { version = "0.6", optional = true } httparse = { version = "1.2", optional = true } +smallvec = { version = "0.6", optional = true } twoway = { version = "0.1", optional = true } # Optional Integrations @@ -40,7 +41,7 @@ env_logger = "0.3" [features] client = [] default = ["all"] -server = ["buf_redux", "httparse", "safemem", "twoway"] +server = ["buf_redux", "httparse", "safemem", "smallvec", "twoway"] mock = [] nightly = [] bench = [] diff --git a/multipart/src/server/field.rs b/multipart/src/server/field.rs index 0cc45ff1f..1ab8175b6 100644 --- a/multipart/src/server/field.rs +++ b/multipart/src/server/field.rs @@ -55,7 +55,6 @@ where R: BufRead, F: FnOnce(&[StrHeader]) -> Ret { return Err(ParseHeaderError::Other("Could not read field headers".to_string())); } - // FIXME: https://github.com/seanmonstar/httparse/issues/34 match httparse::parse_headers(buf, &mut raw_headers) { Ok(Status::Complete((consume_, raw_headers))) => { consume = consume_; @@ -87,11 +86,25 @@ fn copy_headers<'h, 'b: 'h>(raw: &[Header<'b>], headers: &'h mut [StrHeader<'b>] } /// The headers that (may) appear before a `multipart/form-data` field. +/// +/// ### Warning: Values are Client-Provided +/// Everything in this struct are values from the client and should be considered **untrustworthy**. +/// This crate makes no effort to validate or sanitize any client inputs. pub struct FieldHeaders { - /// The `Content-Disposition` header, required. - cont_disp: ContentDisp, - /// The `Content-Type` header, optional. - cont_type: Option, + /// The field's name from the form. + pub name: Arc, + + /// The filename of this entry, if supplied. This is not guaranteed to match the original file + /// or even to be a valid filename for the current platform. + pub filename: Option, + + /// The MIME type (`Content-Type` value) of this file, if supplied by the client. + /// + /// If this is not supplied, the content-type of the field should default to `text/plain` as + /// per [IETF RFC 7578, section 4.4](https://tools.ietf.org/html/rfc7578#section-4.4), but this + /// should not be implicitly trusted. This crate makes no attempt to identify or validate + /// the content-type of the actual field data. + pub content_type: Option, } impl FieldHeaders { @@ -103,8 +116,9 @@ impl FieldHeaders { fn parse(headers: &[StrHeader]) -> Result { let cont_disp = ContentDisp::parse(headers)?.ok_or(ParseHeaderError::MissingContentDisposition)?; Ok(FieldHeaders { - cont_disp: cont_disp, - cont_type: parse_cont_type(headers)?, + name: cont_disp.name, + filename: cont_disp.filename, + content_type: parse_content_type(headers)?, }) } } @@ -159,7 +173,7 @@ impl ContentDisp { } } -fn parse_cont_type(headers: &[StrHeader]) -> Result, ParseHeaderError> { +fn parse_content_type(headers: &[StrHeader]) -> Result, ParseHeaderError> { const CONTENT_TYPE: &'static str = "Content-Type"; let header = if let Some(header) = find_header(headers, CONTENT_TYPE) { header @@ -179,26 +193,33 @@ fn parse_cont_type(headers: &[StrHeader]) -> Result, ParseHeaderErr /// This crate makes no effort to validate or sanitize any client inputs. #[derive(Debug)] pub struct MultipartField { - /// The field's name from the form. - pub name: String, - - /// The filename of this entry, if supplied. This is not guaranteed to match the original file - /// or even to be a valid filename for the current platform. - pub filename: Option, - - /// The MIME type (`Content-Type` value) of this file, if supplied by the client. + /// The headers for this field, including the name, filename, and content-type, if provided. /// - /// If this is not supplied, the content-type of the field should default to `text/plain` as - /// per [IETF RFC 7578, section 4.4](https://tools.ietf.org/html/rfc7578#section-4.4), but this - /// should not be implicitly trusted. This crate makes no attempt to identify or validate - /// the content-type of the actual field data. - pub content_type: Option, + /// ### Warning: Values are Client-Provided + /// Everything in this struct are values from the client and should be considered **untrustworthy**. + /// This crate makes no effort to validate or sanitize any client inputs. + pub headers: FieldHeaders, /// The field's data. pub data: MultipartData, } impl MultipartField { + /// Returns `true` if this field has no content-type or the content-type is `text/plain`. + /// + /// This typically means it can be read to a string, but it could still be using an unsupported + /// character encoding, so decoding to `String` needs to ensure that the data is valid UTF-8. + /// + /// Note also that the field contents may be too large to reasonably fit in memory. + /// The `.save()` adapter can be used to enforce a size limit. + /// + /// Detecting character encodings by any means is (currently) beyond the scope of this crate. + pub fn is_text(&self) -> bool { + self.headers.content_type.as_ref() + .map(|ct| ct.type_() == mime::TEXT && ct.subtype() == mime::PLAIN) + .unwrap_or(true) + } + /// Read the next entry in the request. pub fn next_entry(self) -> ReadEntryResult { self.data.into_inner().read_entry() @@ -230,8 +251,7 @@ impl MultipartField { /// The data of a field in a `multipart/form-data` request. /// -/// You can read it to EOF, or use one of the `save()` method -/// to save it to disk/memory. +/// You can read it to EOF, or use the `save()` adaptor to save it to disk/memory. #[derive(Debug)] pub struct MultipartData { inner: Option, @@ -254,6 +274,10 @@ impl MultipartData where M: ReadEntry { fn into_inner(self) -> M { self.inner.expect("MultipartFile::inner taken!") } + + fn give_inner(&mut self, inner: M) { + self.inner = Some(inner); + } } impl Read for MultipartData { @@ -297,8 +321,8 @@ fn io_str_utf8(buf: &[u8]) -> io::Result<&str> { } fn find_header<'a, 'b>(headers: &'a [StrHeader<'b>], name: &str) -> Option<&'a StrHeader<'b>> { - /// Field names are case insensitive and consist of ASCII characters - /// only (see https://tools.ietf.org/html/rfc822#section-3.2). + // Field names are case insensitive and consist of ASCII characters + // only (see https://tools.ietf.org/html/rfc822#section-3.2). headers.iter().find(|header| header.name.eq_ignore_ascii_case(name)) } @@ -312,44 +336,27 @@ pub trait ReadEntry: PrivReadEntry + Sized { return End(self); } - let field_headers = try_read_entry!(self; self.read_headers()); + let field_headers: FieldHeaders = try_read_entry!(self; self.read_headers()); - let data = match field_headers.cont_type { - Some(cont_type) => { - match cont_type.0 { - TopLevel::Multipart => { - let msg = format!("Error on field {:?}: nested multipart fields are \ + match field_headers.cont_type { + Some(ref cont_type) if cont_type.type_() == mime::MULTIPART => { + let msg = format!("Error on field {:?}: nested multipart fields are \ not supported. However, reports of clients sending \ requests like this are welcome at \ https://github.com/abonander/multipart/issues/56", - field_headers.cont_disp.field_name); - - return ReadEntryResult::invalid_data(self, msg); - }, - _ => { - MultipartData::File( - MultipartFile { - filename: field_headers.cont_disp.filename, - content_type: cont_type, - inner: Some(self) - } - ) - } - } - }, - None => { - let text = try_read_entry!(self; self.read_to_string()); - MultipartData::Text(MultipartText { - text: text, - inner: Some(self), - }) + field_headers.cont_disp.field_name); + + return ReadEntryResult::invalid_data(self, msg); }, - }; + _ => (), + } Entry( MultipartField { - name: field_headers.cont_disp.field_name, - data: data, + headers: field_headers, + data: MultipartData { + inner: Some(self), + }, } ) } diff --git a/multipart/src/server/save.rs b/multipart/src/server/save.rs index 227c84de6..cb60ff46e 100644 --- a/multipart/src/server/save.rs +++ b/multipart/src/server/save.rs @@ -20,6 +20,8 @@ use std::fs::OpenOptions; use std::path::{Path, PathBuf}; use std::{env, fs, io, mem}; +use server::field::FieldHeaders; + const RANDOM_FILENAME_LEN: usize = 12; fn rand_filename() -> String { @@ -187,7 +189,7 @@ impl SaveBuilder where M: ReadEntry { Some(limit) if count >= limit => return Partial ( PartialEntries { entries: entries, - partial_file: Some(PartialFileField { + partial_file: Some(PartialSavedField { field_name: field.name, source: file, dest: None, @@ -208,7 +210,7 @@ impl SaveBuilder where M: ReadEntry { Partial(partial, reason) => return Partial( PartialEntries { entries: entries, - partial_file: Some(PartialFileField { + partial_file: Some(PartialSavedField { field_name: field.name, source: file, dest: Some(partial) @@ -219,7 +221,7 @@ impl SaveBuilder where M: ReadEntry { Error(e) => return Partial( PartialEntries { entries: entries, - partial_file: Some(PartialFileField { + partial_file: Some(PartialSavedField { field_name: field.name, source: file, dest: None, @@ -241,7 +243,7 @@ impl SaveBuilder where M: ReadEntry { } /// Save API for individual files. -impl<'m, M: 'm> SaveBuilder<&'m mut MultipartFile> where MultipartFile: BufRead { +impl<'m, M: 'm> SaveBuilder<&'m mut MultipartData> where MultipartData: BufRead { /// Save to a file with a random alphanumeric name in the OS temporary directory. /// @@ -257,17 +259,7 @@ impl<'m, M: 'm> SaveBuilder<&'m mut MultipartFile> where MultipartFile: Bu /// /// See `with_path()` for more details. /// - /// ### Warning: Do **not* trust user input! - /// It is a serious security risk to create files or directories with paths based on user input. - /// A malicious user could craft a path which can be used to overwrite important files, such as - /// web templates, static assets, Javascript files, database files, configuration files, etc., - /// if they are writable by the server process. - /// - /// This can be mitigated somewhat by setting filesystem permissions as - /// conservatively as possible and running the server under its own user with restricted - /// permissions, but you should still not use user input directly as filesystem paths. - /// If it is truly necessary, you should sanitize filenames such that they cannot be - /// misinterpreted by the OS. + pub fn with_filename(&mut self, filename: &str) -> FileSaveResult { let mut tempdir = env::temp_dir(); tempdir.set_file_name(filename); @@ -278,18 +270,6 @@ impl<'m, M: 'm> SaveBuilder<&'m mut MultipartFile> where MultipartFile: Bu /// Save to a file with a random alphanumeric name in the given directory. /// /// See `with_path()` for more details. - /// - /// ### Warning: Do **not* trust user input! - /// It is a serious security risk to create files or directories with paths based on user input. - /// A malicious user could craft a path which can be used to overwrite important files, such as - /// web templates, static assets, Javascript files, database files, configuration files, etc., - /// if they are writable by the server process. - /// - /// This can be mitigated somewhat by setting filesystem permissions as - /// conservatively as possible and running the server under its own user with restricted - /// permissions, but you should still not use user input directly as filesystem paths. - /// If it is truly necessary, you should sanitize filenames such that they cannot be - /// misinterpreted by the OS. pub fn with_dir>(&mut self, dir: P) -> FileSaveResult { let path = dir.as_ref().join(rand_filename()); self.with_path(path) @@ -347,37 +327,10 @@ impl<'m, M: 'm> SaveBuilder<&'m mut MultipartFile> where MultipartFile: Bu /// A file saved to the local filesystem from a multipart request. #[derive(Debug)] pub struct SavedFile { + /// The headers of the field that was saved. + pub headers: FieldHeaders, /// The complete path this file was saved at. pub path: PathBuf, - - /// ### Warning: Client Provided / Untrustworthy - /// You should treat this value as **untrustworthy** because it is an arbitrary string - /// provided by the client. - /// - /// It is a serious security risk to create files or directories with paths based on user input. - /// A malicious user could craft a path which can be used to overwrite important files, such as - /// web templates, static assets, Javascript files, database files, configuration files, etc., - /// if they are writable by the server process. - /// - /// This can be mitigated somewhat by setting filesystem permissions as - /// conservatively as possible and running the server under its own user with restricted - /// permissions, but you should still not use user input directly as filesystem paths. - /// If it is truly necessary, you should sanitize filenames such that they cannot be - /// misinterpreted by the OS. Such functionality is outside the scope of this crate. - pub filename: Option, - - /// The MIME type (`Content-Type` value) of this file, if supplied by the client, - /// or `"applicaton/octet-stream"` otherwise. - /// - /// ### Note: Client Provided - /// Consider this value to be potentially untrustworthy, as it is provided by the client. - /// It may be inaccurate or entirely wrong, depending on how the client determined it. - /// - /// Some variants wrap arbitrary strings which could be abused by a malicious user if your - /// application performs any non-idempotent operations based on their value, such as - /// starting another program or querying/updating a database (web-search "SQL injection"). - pub content_type: Mime, - /// The number of bytes written to the disk. pub size: u64, } @@ -388,67 +341,12 @@ impl SavedFile { } } -/// Wrapper for the fields of an `Entries`. Stores text fields in a `Vec<(String, String)>` to -/// enable accessing fields with duplicated names. - -#[derive(Debug, Default)] -pub struct FieldsWrapper(Vec<(String, String)>); - -impl FieldsWrapper { - - /// Converts the contents into a multivalued hashmap - pub fn into_multivalued_map(self) -> HashMap> { - let mut out = HashMap::new(); - for (key, val) in self.0.into_iter() { - out.entry(key).or_insert_with(Vec::new).push(val); - } - out - } - - /// Converts the contents into a single-valued hashmap - pub fn into_map(self) -> HashMap { - self.0.into_iter().collect() - } -} - -impl IntoIterator for FieldsWrapper { - type Item = (String, String); - type IntoIter = ::std::vec::IntoIter; - - fn into_iter(self) -> Self::IntoIter { - self.0.into_iter() - } -} - -impl ::std::ops::Deref for FieldsWrapper { - type Target = Vec<(String, String)>; - - fn deref(&self) -> &Self::Target { - &self.0 - } -} - -impl ::std::ops::DerefMut for FieldsWrapper { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.0 - } -} - -// this allows use of both `from(FieldsWrapper)` and `FieldsWrapper.into()` -impl From for Vec<(String, String)> { - fn from(fields: FieldsWrapper) -> Self { - fields.0 - } -} - /// A result of `Multipart::save_all()`. #[derive(Debug)] pub struct Entries { /// The text fields of the multipart request, mapped by field name -> value. - pub fields: FieldsWrapper, - /// A map of file field names to their contents saved on the filesystem. - pub files: HashMap>, - /// The directory the files in this request were saved under; may be temporary or permanent. + pub fields: HashMap, + pub save_dir: SaveDir, } @@ -480,7 +378,7 @@ pub enum SaveDir { /// This directory is permanent and will be left on the filesystem when this wrapper is dropped. /// /// **N.B.** If this directory is in the OS temporary directory then it may still be - /// deleted at any time, usually on reboot or when free space is low. + /// deleted at any time. Perm(PathBuf), } @@ -592,16 +490,12 @@ impl PartialReason { } } -/// The file field that was being read when the save operation quit. +/// The field that was being read when the save operation quit. /// /// May be partially saved to the filesystem if `dest` is `Some`. #[derive(Debug)] -pub struct PartialFileField { - /// The field name for the partial file. - pub field_name: String, - /// The partial file's source in the multipart stream (may be partially read if `dest` - /// is `Some`). - pub source: MultipartFile, +pub struct PartialSavedField { + pub source: MultipartField, /// The partial file's entry on the filesystem, if the operation got that far. pub dest: Option, } @@ -615,9 +509,9 @@ pub struct PartialFileField { pub struct PartialEntries { /// The entries that were saved successfully. pub entries: Entries, - /// The file that was in the process of being read. `None` if the error - /// occurred between file entries. - pub partial_file: Option>, + /// The field that was in the process of being read. `None` if the error + /// occurred between entries. + pub partial_field: Option>, } /// Discards `partial_file` From b232de42b8823d01ec1889fe4538f7faab694768 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sun, 17 Dec 2017 11:36:13 -0800 Subject: [PATCH 351/453] WIP refactoring (2) MAJOR: removed `multipart-nickel` --- multipart/Cargo.toml | 10 +- multipart/README.md | 5 +- multipart/examples/hyper_server.rs | 4 +- multipart/examples/iron.rs | 4 +- multipart/examples/nickel.rs | 92 ++++++++++++++++-- multipart/examples/tiny_http.rs | 4 +- multipart/nickel/.gitignore | 3 - multipart/nickel/Cargo.toml | 21 ---- multipart/nickel/examples/nickel.rs | 84 ---------------- multipart/nickel/src/lib.rs | 125 ------------------------ multipart/src/lib.rs | 13 ++- multipart/src/server/field.rs | 54 +++++----- multipart/src/server/mod.rs | 16 ++- multipart/src/server/nickel.rs | 65 +++++++++++++ multipart/src/server/save.rs | 146 ++++++++++++++++++++++------ 15 files changed, 323 insertions(+), 323 deletions(-) delete mode 100644 multipart/nickel/.gitignore delete mode 100644 multipart/nickel/Cargo.toml delete mode 100644 multipart/nickel/examples/nickel.rs delete mode 100644 multipart/nickel/src/lib.rs create mode 100644 multipart/src/server/nickel.rs diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index cd1d4506f..7d78bbe86 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -27,25 +27,27 @@ clippy = { version = ">=0.0, <0.1", optional = true} #Server Dependencies buf_redux = { version = "0.6", optional = true } httparse = { version = "1.2", optional = true } -smallvec = { version = "0.6", optional = true } twoway = { version = "0.1", optional = true } # Optional Integrations hyper = { version = ">=0.9, <0.11", optional = true, default-features = false } iron = { version = ">=0.4,<0.6", optional = true } tiny_http = { version = "0.5", optional = true } +nickel = { version = ">=0.10.1", optional = true } [dev-dependencies] env_logger = "0.3" [features] client = [] -default = ["all"] -server = ["buf_redux", "httparse", "safemem", "smallvec", "twoway"] +default = ["server"] +server = ["buf_redux", "httparse", "safemem", "twoway"] mock = [] nightly = [] bench = [] # Use this to enable SSE4.2 instructions in boundary finding # TODO: Benchmark this sse4 = ["nightly", "twoway/pcmp"] -all = ["client", "server", "hyper", "iron", "tiny_http", "mock"] +all = ["client", "server", "hyper", "iron", "tiny_http", "nickel", "mock"] +# switch uses of `Arc` (`From` impl only stabilized in 1.21) for `Arc` +no_arc_str = [] diff --git a/multipart/README.md b/multipart/README.md index 67e9c8fa1..34b3ab0d6 100644 --- a/multipart/README.md +++ b/multipart/README.md @@ -31,10 +31,7 @@ via the `tiny_http` feature. Provides server-side integration with `tiny_http::Request` via `multipart::server::Multipart`. -### [Nickel](http://nickel.rs/) - -**Note**: Moved to `multipart-nickel` crate, see [nickel/examples/nickel.rs](nickel/examples/nickel.rs) -for updated integration example. +### [Nickel](http://nickel.rs/) ^(returning to `multipart` in 0.14!) Provides server-side integration with `&mut nickel::Request` via `multipart::server::Multipart`. diff --git a/multipart/examples/hyper_server.rs b/multipart/examples/hyper_server.rs index e3869027e..d141c69fe 100644 --- a/multipart/examples/hyper_server.rs +++ b/multipart/examples/hyper_server.rs @@ -7,7 +7,7 @@ use hyper::server::{Handler, Server, Request, Response}; use hyper::status::StatusCode; use hyper::server::response::Response as HyperResponse; use multipart::server::hyper::{Switch, MultipartHandler, HyperRequest}; -use multipart::server::{Multipart, Entries, SaveResult, SavedFile}; +use multipart::server::{Multipart, Entries, SaveResult, SavedField}; struct NonMultipart; impl Handler for NonMultipart { @@ -57,7 +57,7 @@ fn process_entries<'a>(entries: Entries) -> io::Result<()> { Ok(()) } -fn print_file(saved_file: &SavedFile) -> io::Result<()> { +fn print_file(saved_file: &SavedField) -> io::Result<()> { let mut file = File::open(&saved_file.path)?; let mut contents = String::new(); diff --git a/multipart/examples/iron.rs b/multipart/examples/iron.rs index d09d29f28..9230b19e3 100644 --- a/multipart/examples/iron.rs +++ b/multipart/examples/iron.rs @@ -5,7 +5,7 @@ extern crate env_logger; use std::fs::File; use std::io::Read; -use multipart::server::{Multipart, Entries, SaveResult, SavedFile}; +use multipart::server::{Multipart, Entries, SaveResult, SavedField}; use iron::prelude::*; use iron::status; @@ -62,7 +62,7 @@ fn process_entries(entries: Entries) -> IronResult { Ok(Response::with((status::Ok, "Multipart data is processed"))) } -fn print_file(saved_file: &SavedFile) -> IronResult<()> { +fn print_file(saved_file: &SavedField) -> IronResult<()> { let mut file = match File::open(&saved_file.path) { Ok(file) => file, Err(error) => { diff --git a/multipart/examples/nickel.rs b/multipart/examples/nickel.rs index 78f940567..d05aca846 100644 --- a/multipart/examples/nickel.rs +++ b/multipart/examples/nickel.rs @@ -1,8 +1,84 @@ -//! **Note**: in-crate integration for Nickel was removed in 0.11.0; -//! integration will be provided in the -//! [`multipart-nickel`](https://crates.io/crates/multipart-nickel) -//! crate for the foreseeable future. -//! -//! Please see `nickel/examples/nickel.rs` for the new integration. - -fn main() {} +extern crate multipart; +extern crate nickel; + +use std::fs::File; +use std::io::Read; +use nickel::{HttpRouter, MiddlewareResult, Nickel, Request, Response}; + +use multipart_nickel::MultipartBody; +use multipart_nickel::multipart_server::{Entries, SaveResult}; + +fn handle_multipart<'mw>(req: &mut Request, mut res: Response<'mw>) -> MiddlewareResult<'mw> { + match req.multipart_body() { + Some(mut multipart) => { + match multipart.save().temp() { + SaveResult::Full(entries) => process_entries(res, entries), + + SaveResult::Partial(entries, e) => { + println!("Partial errors ... {:?}", e); + return process_entries(res, entries.keep_partial()); + }, + + SaveResult::Error(e) => { + println!("There are errors in multipart POSTing ... {:?}", e); + res.set(nickel::status::StatusCode::InternalServerError); + return res.send(format!("Server could not handle multipart POST! {:?}", e)); + }, + } + } + None => { + res.set(nickel::status::StatusCode::BadRequest); + return res.send("Request seems not was a multipart request") + } + } +} + +/// Processes saved entries from multipart request. +/// Returns an OK response or an error. +fn process_entries<'mw>(res: Response<'mw>, entries: Entries) -> MiddlewareResult<'mw> { + for (name, field) in entries.fields { + println!("Field {:?}: {:?}", name, field); + } + + for (name, files) in entries.files { + println!("Field {:?} has {} files:", name, files.len()); + + for saved_file in files { + match File::open(&saved_file.path) { + Ok(mut file) => { + let mut contents = String::new(); + if let Err(e) = file.read_to_string(&mut contents) { + println!("Could not read file {:?}. Error: {:?}", saved_file.filename, e); + return res.error(nickel::status::StatusCode::BadRequest, "The uploaded file was not readable") + } + + println!("File {:?} ({:?}):", saved_file.filename, saved_file.content_type); + println!("{}", contents); + file + } + Err(e) => { + println!("Could open file {:?}. Error: {:?}", saved_file.filename, e); + return res.error(nickel::status::StatusCode::BadRequest, "The uploaded file was not readable") + } + }; + } + } + + res.send("Ok") +} + +fn main() { + let mut srv = Nickel::new(); + + srv.post("/multipart_upload/", handle_multipart); + + // Start this example via: + // + // `cargo run --example nickel --features nickel` + // + // And - if you are in the root of this repository - do an example + // upload via: + // + // `curl -F file=@LICENSE 'http://localhost:6868/multipart_upload/'` + srv.listen("127.0.0.1:6868").expect("Failed to bind server"); +} diff --git a/multipart/examples/tiny_http.rs b/multipart/examples/tiny_http.rs index 9571b8e82..101927732 100644 --- a/multipart/examples/tiny_http.rs +++ b/multipart/examples/tiny_http.rs @@ -3,7 +3,7 @@ extern crate multipart; use std::fs::File; use std::io::{self, Read}; -use multipart::server::{Multipart, Entries, SaveResult, SavedFile}; +use multipart::server::{Multipart, Entries, SaveResult, SavedField}; use tiny_http::{Response, StatusCode, Request}; fn main() { // Starting a server on `localhost:80` @@ -67,7 +67,7 @@ fn process_entries<'a>(entries: Entries) -> io::Result> { Ok(build_response(200, "Multipart data is received!")) } -fn print_file(saved_file: &SavedFile) -> io::Result<()> { +fn print_file(saved_file: &SavedField) -> io::Result<()> { let mut file = File::open(&saved_file.path)?; let mut contents = String::new(); diff --git a/multipart/nickel/.gitignore b/multipart/nickel/.gitignore deleted file mode 100644 index c3ad9545f..000000000 --- a/multipart/nickel/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -target/ -*.iml -Cargo.lock diff --git a/multipart/nickel/Cargo.toml b/multipart/nickel/Cargo.toml deleted file mode 100644 index 635673bd6..000000000 --- a/multipart/nickel/Cargo.toml +++ /dev/null @@ -1,21 +0,0 @@ -[package] -name = "multipart-nickel" -version = "0.3.0" -authors = ["Austin Bonander "] - -description = "Support for `multipart/form-data` bodies in Nickel via the `multipart` crate." -documentation = "https://docs.rs/multipart-nickel" -repository = "https://github.com/abonander/multipart" -license = "MIT OR Apache-2.0" - -[dependencies] -hyper = "0.10" -nickel = "0.10" - -[dependencies.multipart] -version = "0.13" -# You can specify a path alongside the version so it can be tested against the latest code but published -# against the released version -path = "../" -default-features = false -features = ["hyper", "server"] diff --git a/multipart/nickel/examples/nickel.rs b/multipart/nickel/examples/nickel.rs deleted file mode 100644 index f252c331a..000000000 --- a/multipart/nickel/examples/nickel.rs +++ /dev/null @@ -1,84 +0,0 @@ -extern crate multipart_nickel; -extern crate nickel; - -use std::fs::File; -use std::io::Read; -use nickel::{HttpRouter, MiddlewareResult, Nickel, Request, Response}; - -use multipart_nickel::MultipartBody; -use multipart_nickel::multipart_server::{Entries, SaveResult}; - -fn handle_multipart<'mw>(req: &mut Request, mut res: Response<'mw>) -> MiddlewareResult<'mw> { - match req.multipart_body() { - Some(mut multipart) => { - match multipart.save().temp() { - SaveResult::Full(entries) => process_entries(res, entries), - - SaveResult::Partial(entries, e) => { - println!("Partial errors ... {:?}", e); - return process_entries(res, entries.keep_partial()); - }, - - SaveResult::Error(e) => { - println!("There are errors in multipart POSTing ... {:?}", e); - res.set(nickel::status::StatusCode::InternalServerError); - return res.send(format!("Server could not handle multipart POST! {:?}", e)); - }, - } - } - None => { - res.set(nickel::status::StatusCode::BadRequest); - return res.send("Request seems not was a multipart request") - } - } -} - -/// Processes saved entries from multipart request. -/// Returns an OK response or an error. -fn process_entries<'mw>(res: Response<'mw>, entries: Entries) -> MiddlewareResult<'mw> { - for (name, field) in entries.fields { - println!("Field {:?}: {:?}", name, field); - } - - for (name, files) in entries.files { - println!("Field {:?} has {} files:", name, files.len()); - - for saved_file in files { - match File::open(&saved_file.path) { - Ok(mut file) => { - let mut contents = String::new(); - if let Err(e) = file.read_to_string(&mut contents) { - println!("Could not read file {:?}. Error: {:?}", saved_file.filename, e); - return res.error(nickel::status::StatusCode::BadRequest, "The uploaded file was not readable") - } - - println!("File {:?} ({:?}):", saved_file.filename, saved_file.content_type); - println!("{}", contents); - file - } - Err(e) => { - println!("Could open file {:?}. Error: {:?}", saved_file.filename, e); - return res.error(nickel::status::StatusCode::BadRequest, "The uploaded file was not readable") - } - }; - } - } - - res.send("Ok") -} - -fn main() { - let mut srv = Nickel::new(); - - srv.post("/multipart_upload/", handle_multipart); - - // Start this example via: - // - // `cargo run --example nickel --features nickel` - // - // And - if you are in the root of this repository - do an example - // upload via: - // - // `curl -F file=@LICENSE 'http://localhost:6868/multipart_upload/'` - srv.listen("127.0.0.1:6868").expect("Failed to bind server"); -} diff --git a/multipart/nickel/src/lib.rs b/multipart/nickel/src/lib.rs deleted file mode 100644 index 20e048769..000000000 --- a/multipart/nickel/src/lib.rs +++ /dev/null @@ -1,125 +0,0 @@ -//! Support for `multipart/form-data` bodies in [Nickel](https://nickel.rs) via the -//! [`multipart`](https://crates.io/crates/multipart) crate. -//! -//! ### Why an external crate? -//! Three major reasons led to the decision to move Nickel integration to an external crate. -//! -//! 1: The part of Nickel's public API that matters to `multipart` (getting headers and -//! reading the request) uses Hyper's request type; this means that Nickel's integration -//! must necessarily be coupled to Hyper's integration. -//! -//! 2: Cargo does not allow specifying two different versions of the same crate in the -//! same manifest, probably for some good reasons but this crate's author has not looked into it. -//! -//! 3: Nickel's development moves incredibly slowly; when a new version of Hyper releases, there is -//! always a significant lag before Nickel upgrades, -//! [sometimes several months](https://github.com/nickel-org/nickel.rs/issues/367)--in this case, -//! Hyper was upgraded (in May) two months before the issue was opened (July), but a new version of -//! Nickel was not published until four months later (September). -//! -//! This causes problems for `multipart` because it cannot upgrade Hyper until Nickel does, -//! but its Hyper users often want to upgrade their Hyper as soon as possible. -//! -//! In order to provide up-to-date integration for Hyper, it was necessary to move Nickel -//! integration to an external crate so it can be pinned at the version of Hyper that Nickel -//! supports. This allows `multipart` to upgrade Hyper arbitrarily and still keep everyone happy. -//! -//! ### Porting from `multipart`'s Integration -//! -//! Whereas `multipart` only provided one way to wrap a Nickel request, this crate provides two: -//! -//! * To continue using `Multipart::from_request()`, wrap the request in -//! [`Maybe`](struct.Maybe.html): -//! -//! ```ignore -//! // Where `req` is `&mut nickel::Request` -//! - Multipart::from_request(req) -//! + use multipart_nickel::Maybe; -//! + Multipart::from_request(Maybe(req)) -//! ``` -//! -//! * Import `multipart_nickel::MultipartBody` and call `.multipart_body()`, which returns -//! `Option` (which better matches the conventions of `nickel::FormBody` and `nickel::JsonBody`): -//! -//! ```rust,ignore -//! use multipart_nickel::MultipartBody; -//! -//! // Where `req` is `&mut nickel::Request` -//! match req.multipart_body() { -//! Some(multipart) => // handle multipart body -//! None => // handle regular body -//! } -//! ``` -//! -//! This crate also reexports the `server` module from the version of `multipart` -//! that it uses, so that you don't have to import `multipart` separately. -extern crate hyper; -extern crate multipart; -extern crate nickel; - -use nickel::Request as NickelRequest; - -use hyper::server::Request as HyperRequest; - -/// The `server` module from the version of `multipart` that this crate uses. -pub use multipart::server as multipart_server; - -use multipart_server::{HttpRequest, Multipart}; - -/// A wrapper for `&mut nickel::Request` which implements `multipart::server::HttpRequest`. -/// -/// Necessary because this crate cannot directly provide an impl of `HttpRequest` for -/// `&mut NickelRequest`. -pub struct Maybe<'r, 'mw: 'r, 'server: 'mw, D: 'mw>(pub &'r mut NickelRequest<'mw, 'server, D>); - -impl<'r, 'mw: 'r, 'server: 'mw, D: 'mw> HttpRequest for Maybe<'r, 'mw, 'server, D> { - type Body = &'r mut HyperRequest<'mw, 'server>; - - fn multipart_boundary(&self) -> Option<&str> { - self.0.origin.multipart_boundary() - } - - fn body(self) -> Self::Body { - &mut self.0.origin - } -} - -/// Extension trait for getting the `multipart/form-data` body from `nickel::Request`. -/// -/// Implemented for `nickel::Request`. -pub trait MultipartBody<'mw, 'server> { - /// Get a multipart reader for the request body, if the request is of the right type. - fn multipart_body(&mut self) -> Option>>; -} - -impl<'mw, 'server, D: 'mw> MultipartBody<'mw, 'server> for NickelRequest<'mw, 'server, D> { - fn multipart_body(&mut self) -> Option>> { - Multipart::from_request(Maybe(self)).ok() - } -} - - -impl<'r, 'mw: 'r, 'server: 'mw, D: 'mw> AsRef<&'r mut NickelRequest<'mw, 'server, D>> for Maybe<'r, 'mw, 'server, D> { - fn as_ref(&self) -> &&'r mut NickelRequest<'mw, 'server, D> { - &self.0 - } -} - -impl<'r, 'mw: 'r, 'server: 'mw, D: 'mw> AsMut<&'r mut NickelRequest<'mw, 'server, D>> for Maybe<'r, 'mw, 'server, D> { - fn as_mut(&mut self) -> &mut &'r mut NickelRequest<'mw, 'server, D> { - &mut self.0 - } -} - -impl<'r, 'mw: 'r, 'server: 'mw, D: 'mw> Into<&'r mut NickelRequest<'mw, 'server, D>> for Maybe<'r, 'mw, 'server, D> { - fn into(self) -> &'r mut NickelRequest<'mw, 'server, D> { - self.0 - } -} - -impl<'r, 'mw: 'r, 'server: 'mw, D: 'mw> From<&'r mut NickelRequest<'mw, 'server, D>> for Maybe<'r, 'mw, 'server, D> { - fn from(req: &'r mut NickelRequest<'mw, 'server, D>) -> Self { - Maybe(req) - } -} - diff --git a/multipart/src/lib.rs b/multipart/src/lib.rs index d23b6fa16..97e01106e 100644 --- a/multipart/src/lib.rs +++ b/multipart/src/lib.rs @@ -25,10 +25,15 @@ //! * `tiny_http`: Integration with the [`tiny_http`](https://github.com/frewsxcv/tiny-http) //! crate. See the [`server::tiny_http`](server/tiny_http/index.html) module for more information. //! -//! **Note**: in-crate integration for Nickel was removed in 0.11.0; -//! integration will be provided in the -//! [`multipart-nickel`](https://crates.io/crates/multipart-nickel) -//! crate for the foreseeable future. +//! `nickel` (returning in 0.14!): Integration with the See the [`server::nickel`](server/nickel/index.html) module for more information. +//! +//! ### Note: Work in Progress +//! I have left a number of Request-for-Comments (RFC) questions on various APIs and other places +//! in the code as there are some cases where I'm not sure what the desirable behavior is. +//! +//! I have opened an issue as a place to collect responses and discussions for these questions +//! [on Github](https://github.com/abonander/multipart/issues/96). Please quote the RFC-statement +//! (and/or link to its source line) and provide your feedback there. #![cfg_attr(feature="clippy", feature(plugin))] #![cfg_attr(feature="clippy", plugin(clippy))] #![cfg_attr(feature="clippy", deny(clippy))] diff --git a/multipart/src/server/field.rs b/multipart/src/server/field.rs index 1ab8175b6..0cbfc3162 100644 --- a/multipart/src/server/field.rs +++ b/multipart/src/server/field.rs @@ -6,14 +6,7 @@ // copied, modified, or distributed except according to those terms. //! `multipart` field header parsing. - -use super::httparse::{self, EMPTY_HEADER, Header, Status}; - -use self::ReadEntryResult::*; - -use super::save::{SaveBuilder, SavedFile}; - -use mime::{TopLevel, Mime}; +use mime::Mime; use std::io::{self, Read, BufRead, Write}; use std::ops::Deref; @@ -22,6 +15,15 @@ use std::{str, fmt, error}; use std::ascii::AsciiExt; +use super::httparse::{self, EMPTY_HEADER, Header, Status}; + +use self::ReadEntryResult::*; + +use super::save::{SaveBuilder, SavedField}; + +use super::ArcStr; + + const EMPTY_STR_HEADER: StrHeader<'static> = StrHeader { name: "", val: "", @@ -90,9 +92,10 @@ fn copy_headers<'h, 'b: 'h>(raw: &[Header<'b>], headers: &'h mut [StrHeader<'b>] /// ### Warning: Values are Client-Provided /// Everything in this struct are values from the client and should be considered **untrustworthy**. /// This crate makes no effort to validate or sanitize any client inputs. +#[derive(Clone, Debug)] pub struct FieldHeaders { /// The field's name from the form. - pub name: Arc, + pub name: ArcStr, /// The filename of this entry, if supplied. This is not guaranteed to match the original file /// or even to be a valid filename for the current platform. @@ -187,10 +190,6 @@ fn parse_content_type(headers: &[StrHeader]) -> Result, ParseHeader } /// A field in a multipart request with its associated headers and data. -/// -/// ### Warning: Values are Client-Provided -/// Everything in this struct are values from the client and should be considered **untrustworthy**. -/// This crate makes no effort to validate or sanitize any client inputs. #[derive(Debug)] pub struct MultipartField { /// The headers for this field, including the name, filename, and content-type, if provided. @@ -205,7 +204,7 @@ pub struct MultipartField { } impl MultipartField { - /// Returns `true` if this field has no content-type or the content-type is `text/plain`. + /// Returns `true` if this field has no content-type or the content-type is `text/...`. /// /// This typically means it can be read to a string, but it could still be using an unsupported /// character encoding, so decoding to `String` needs to ensure that the data is valid UTF-8. @@ -215,9 +214,7 @@ impl MultipartField { /// /// Detecting character encodings by any means is (currently) beyond the scope of this crate. pub fn is_text(&self) -> bool { - self.headers.content_type.as_ref() - .map(|ct| ct.type_() == mime::TEXT && ct.subtype() == mime::PLAIN) - .unwrap_or(true) + self.headers.content_type.as_ref().map_or(true, |ct| ct.type_() == mime::TEXT) } /// Read the next entry in the request. @@ -258,7 +255,7 @@ pub struct MultipartData { } impl MultipartData where M: ReadEntry { - /// Get a builder type which can save the file with or without a size limit. + /// Get a builder type which can save the field with or without a size limit. pub fn save(&mut self) -> SaveBuilder<&mut Self> { SaveBuilder::new(self) } @@ -338,18 +335,15 @@ pub trait ReadEntry: PrivReadEntry + Sized { let field_headers: FieldHeaders = try_read_entry!(self; self.read_headers()); - match field_headers.cont_type { - Some(ref cont_type) if cont_type.type_() == mime::MULTIPART => { - let msg = format!("Error on field {:?}: nested multipart fields are \ - not supported. However, reports of clients sending \ - requests like this are welcome at \ - https://github.com/abonander/multipart/issues/56", - field_headers.cont_disp.field_name); - - return ReadEntryResult::invalid_data(self, msg); - }, - _ => (), - } + field_headers.content_type.as_ref().map(|ct| if ct.type_() == mime::MULTIPART { + // fields of this type are sent by (supposedly) no known clients + // (https://tools.ietf.org/html/rfc7578#appendix-A) so I'd be fascinated + // to hear about any in the wild + note!("Found nested multipart field: {:?}:\r\n\ + Please report this client's User-Agent and any other available details \ + at https://github.com/abonander/multipart/issues/56", + field_headers); + }); Entry( MultipartField { diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index ffafe040a..2f550f938 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -18,6 +18,7 @@ extern crate twoway; use std::borrow::Borrow; use std::io::prelude::*; use std::path::Path; +use std::sync::Arc; use std::io; use tempdir::TempDir; @@ -26,14 +27,25 @@ use self::boundary::BoundaryReader; use self::field::PrivReadEntry; -pub use self::field::{MultipartField, MultipartFile, MultipartData, ReadEntry, ReadEntryResult}; +pub use self::field::{MultipartField, MultipartData, ReadEntry, ReadEntryResult}; use self::save::SaveBuilder; -pub use self::save::{Entries, SaveResult, SavedFile}; +pub use self::save::{Entries, SaveResult, SavedField}; use self::save::EntriesSaveResult; +/// Replacement typedef for `Arc` for older Rust releases. +#[cfg(feature = "no_arc_str")] +pub type ArcStr = Arc; + +/// Typedef for `Arc`. +/// +/// Construction of `Arc` was only stabilized in Rust 1.21, so to continue to support +/// older versions, an alternate typedef of `Arc` is available under the `no_arc_str` feature. +#[cfg(not(feature = "no_arc_str"))] +pub type ArcStr = Arc; + macro_rules! try_opt ( ($expr:expr) => ( match $expr { diff --git a/multipart/src/server/nickel.rs b/multipart/src/server/nickel.rs new file mode 100644 index 000000000..fa9f1fca2 --- /dev/null +++ b/multipart/src/server/nickel.rs @@ -0,0 +1,65 @@ +//! Support for `multipart/form-data` bodies in [Nickel](https://nickel.rs). +extern crate nickel; + +use self::nickel::Request as NickelRequest; +use self::nickel::hyper::server::Request as HyperRequest; + +use server::{HttpRequest, Multipart}; + +/// A wrapper for `&mut nickel::Request` which implements `multipart::server::HttpRequest`. +/// +/// Necessary because this crate cannot directly provide an impl of `HttpRequest` for +/// `&mut NickelRequest`. +pub struct Maybe<'r, 'mw: 'r, 'server: 'mw, D: 'mw>(pub &'r mut NickelRequest<'mw, 'server, D>); + +impl<'r, 'mw: 'r, 'server: 'mw, D: 'mw> HttpRequest for Maybe<'r, 'mw, 'server, D> { + type Body = &'r mut HyperRequest<'mw, 'server>; + + fn multipart_boundary(&self) -> Option<&str> { + self.0.origin.multipart_boundary() + } + + fn body(self) -> Self::Body { + &mut self.0.origin + } +} + +/// Extension trait for getting the `multipart/form-data` body from `nickel::Request`. +/// +/// Implemented for `nickel::Request`. +pub trait MultipartBody<'mw, 'server> { + /// Get a multipart reader for the request body, if the request is of the right type. + fn multipart_body(&mut self) -> Option>>; +} + +impl<'mw, 'server, D: 'mw> MultipartBody<'mw, 'server> for NickelRequest<'mw, 'server, D> { + fn multipart_body(&mut self) -> Option>> { + Multipart::from_request(Maybe(self)).ok() + } +} + + +impl<'r, 'mw: 'r, 'server: 'mw, D: 'mw> AsRef<&'r mut NickelRequest<'mw, 'server, D>> for Maybe<'r, 'mw, 'server, D> { + fn as_ref(&self) -> &&'r mut NickelRequest<'mw, 'server, D> { + &self.0 + } +} + +impl<'r, 'mw: 'r, 'server: 'mw, D: 'mw> AsMut<&'r mut NickelRequest<'mw, 'server, D>> for Maybe<'r, 'mw, 'server, D> { + fn as_mut(&mut self) -> &mut &'r mut NickelRequest<'mw, 'server, D> { + &mut self.0 + } +} + +impl<'r, 'mw: 'r, 'server: 'mw, D: 'mw> Into<&'r mut NickelRequest<'mw, 'server, D>> for Maybe<'r, 'mw, 'server, D> { + fn into(self) -> &'r mut NickelRequest<'mw, 'server, D> { + self.0 + } +} + +impl<'r, 'mw: 'r, 'server: 'mw, D: 'mw> From<&'r mut NickelRequest<'mw, 'server, D>> for Maybe<'r, 'mw, 'server, D> { + fn from(req: &'r mut NickelRequest<'mw, 'server, D>) -> Self { + Maybe(req) + } +} + diff --git a/multipart/src/server/save.rs b/multipart/src/server/save.rs index cb60ff46e..822394a21 100644 --- a/multipart/src/server/save.rs +++ b/multipart/src/server/save.rs @@ -8,19 +8,18 @@ use mime::Mime; -use super::field::{MultipartData, MultipartFile, ReadEntry, ReadEntryResult}; - -use self::SaveResult::*; - pub use tempdir::TempDir; use std::collections::HashMap; use std::io::prelude::*; -use std::fs::OpenOptions; +use std::fs::{self, File, OpenOptions}; use std::path::{Path, PathBuf}; -use std::{env, fs, io, mem}; +use std::{env, io, mem}; -use server::field::FieldHeaders; +use server::field::{FieldHeaders, MultipartField, MultipartData, ReadEntry, ReadEntryResult}; +use server::ArcStr; + +use self::SaveResult::*; const RANDOM_FILENAME_LEN: usize = 12; @@ -37,6 +36,19 @@ macro_rules! try_start ( ) ); +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum TextPolicy { + /// Attempt to read a text field as text, falling back to binary on error + Try, + /// Attempt to read a text field as text, returning any errors + Force, + /// Don't try to read text + Ignore +} + +// 8 MiB, reasonable? +const DEFAULT_MEMORY_THRESHOLD: usize = 8 * 1024 * 1024; + /// A builder for saving a file or files to the local filesystem. /// /// ### `OpenOptions` @@ -75,6 +87,8 @@ pub struct SaveBuilder { open_opts: OpenOptions, size_limit: Option, count_limit: Option, + memory_threshold: usize, + text_policy: TextPolicy, } impl SaveBuilder { @@ -89,6 +103,8 @@ impl SaveBuilder { open_opts: open_opts, size_limit: None, count_limit: None, + memory_threshold: DEFAULT_MEMORY_THRESHOLD, + text_policy: TextPolicy::Try, } } @@ -109,6 +125,43 @@ impl SaveBuilder { self.open_opts.write(true); self } + + /// Set the threshold at which to switch from copying a field into memory to copying + /// it to disk. + /// + /// If `0`, forces fields to save directly to the filesystem. + /// If `usize::MAX`, effectively forces fields to always save to memory. + /// + /// (RFC: usize::MAX` is technically reachable on 32-bit systems, should we test for capacity + /// overflow and switch to disk then? What about if/when `Vec::try_reserve()` becomes a thing?) + pub fn memory_threshold(self, memory_threshold: usize) -> Self { + Self { memory_threshold, ..self } + } + + /// When encountering a field that is apparently text, try to read it to a string or fall + /// back to binary otherwise. + /// + /// Has no effect once `memory_threshold` has been reached. + pub fn try_text(self) -> Self { + Self { text_policy: TextPolicy::Try, ..self } + } + + /// When encountering a field that is apparently text, read it to a string or return an error. + /// + /// (RFC: should this continue to validate UTF-8 when writing to the filesystem?) + pub fn force_text(self) -> Self { + Self { text_policy: TextPolicy::Force, ..self} + } + + /// Don't try to read or validate any field data as UTF-8. + pub fn ignore_text(self) -> Self { + Self { text_policy: TextPolicy::Ignore, ..self } + } + + fn fork(&self, savable: M_) -> Self { + // this actually forking works + Self { savable, .. *self } + } } /// Save API for whole multipart requests. @@ -283,14 +336,12 @@ impl<'m, M: 'm> SaveBuilder<&'m mut MultipartData> where MultipartData: Bu pub fn with_path>(&mut self, path: P) -> FileSaveResult { let path = path.into(); - let saved = SavedFile { - content_type: self.savable.content_type.clone(), - filename: self.savable.filename.clone(), - path: path, - size: 0, + let saved = SavedField { + headers: self.savable.headers.clone(), + data: Data }; - let file = match create_dir_all(&saved.path).and_then(|_| self.open_opts.open(&saved.path)) { + let file = match create_dir_all(&path).and_then(|_| self.open_opts.open(&path)) { Ok(file) => file, Err(e) => return Partial(saved, e.into()) }; @@ -324,37 +375,68 @@ impl<'m, M: 'm> SaveBuilder<&'m mut MultipartData> where MultipartData: Bu } } -/// A file saved to the local filesystem from a multipart request. +/// A saved field (to memory or filesystem) from a multipart request. #[derive(Debug)] -pub struct SavedFile { +pub struct SavedField { /// The headers of the field that was saved. pub headers: FieldHeaders, - /// The complete path this file was saved at. - pub path: PathBuf, - /// The number of bytes written to the disk. - pub size: u64, + /// The data of the field which may reside in-memory or on the filesystem. + pub data: SavedData, } -impl SavedFile { - fn with_size(self, size: u64) -> Self { - SavedFile { size: size, .. self } +#[derive(Debug)] +pub enum SavedData { + /// Data in the form of a Rust string. + Text(String), + Bytes(Vec), + /// A path to a file on the filesystem and its size as written by `multipart`. + File(PathBuf, u64), +} + +impl SavedData { + pub fn readable(&self) -> io::Result { + use self::SavedData::*; + + match *self { + Text(ref text) => Ok(DataReader::Bytes(text.as_ref())), + Bytes(ref bytes) => Ok(DataReader::Bytes(bytes)), + File(ref path, _) => File::open(path), + } + } + + pub fn size(&self) -> u64 { + use self::SavedData::*; + + match *self { + Text(ref text) => text.len() as u64, + Bytes(ref bytes) => bytes.len() as u64, + File(_, size) => size, + } } } +pub enum DataReader<'a> { + Bytes(&'a [u8]), + File(File), +} + +imp + /// A result of `Multipart::save_all()`. #[derive(Debug)] pub struct Entries { - /// The text fields of the multipart request, mapped by field name -> value. - pub fields: HashMap, + /// The fields of the multipart request, mapped by field name -> value. + /// + /// Each vector is guaranteed not to be empty unless externally modified. + pub fields: HashMap>, - pub save_dir: SaveDir, + pub save_dir: Option, } impl Entries { fn new(save_dir: SaveDir) -> Self { Entries { - fields: FieldsWrapper::default(), - files: HashMap::new(), + fields: HashMap::new(), save_dir: save_dir, } } @@ -364,7 +446,7 @@ impl Entries { self.fields.is_empty() && self.files.is_empty() } - fn mut_files_for(&mut self, field: String) -> &mut Vec { + fn mut_files_for(&mut self, field: String) -> &mut Vec { self.files.entry(field).or_insert_with(Vec::new) } } @@ -497,7 +579,7 @@ impl PartialReason { pub struct PartialSavedField { pub source: MultipartField, /// The partial file's entry on the filesystem, if the operation got that far. - pub dest: Option, + pub dest: Option, } /// The partial result type for `Multipart::save*()`. @@ -552,12 +634,12 @@ pub enum SaveResult { /// Shorthand result for methods that return `Entries` pub type EntriesSaveResult = SaveResult>; -/// Shorthand result for methods that return `SavedFile`s. +/// Shorthand result for methods that return `FieldData`s. /// -/// The `MultipartFile` is not provided here because it is not necessary to return +/// The `MultipartData` is not provided here because it is not necessary to return /// a borrow when the owned version is probably in the same scope. This hopefully /// saves some headache with the borrow-checker. -pub type FileSaveResult = SaveResult; +pub type FieldSaveResult = SaveResult; impl EntriesSaveResult { /// Take the `Entries` from `self`, if applicable, and discarding From ac9b84dc70726a9efd68e15107d626e9a076651b Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Mon, 18 Dec 2017 20:47:10 -0800 Subject: [PATCH 352/453] WIP complete-ish save API, fixing compiler errors --- multipart/src/server/field.rs | 6 +- multipart/src/server/save.rs | 158 ++++++++++++++++++++++++++-------- 2 files changed, 123 insertions(+), 41 deletions(-) diff --git a/multipart/src/server/field.rs b/multipart/src/server/field.rs index 0cbfc3162..6f8c7ed7c 100644 --- a/multipart/src/server/field.rs +++ b/multipart/src/server/field.rs @@ -6,7 +6,7 @@ // copied, modified, or distributed except according to those terms. //! `multipart` field header parsing. -use mime::Mime; +use mime::{self, Mime}; use std::io::{self, Read, BufRead, Write}; use std::ops::Deref; @@ -127,7 +127,7 @@ impl FieldHeaders { } /// The `Content-Disposition` header. -pub struct ContentDisp { +struct ContentDisp { /// The name of the `multipart/form-data` field. field_name: String, /// The optional filename for this field. @@ -339,7 +339,7 @@ pub trait ReadEntry: PrivReadEntry + Sized { // fields of this type are sent by (supposedly) no known clients // (https://tools.ietf.org/html/rfc7578#appendix-A) so I'd be fascinated // to hear about any in the wild - note!("Found nested multipart field: {:?}:\r\n\ + info!("Found nested multipart field: {:?}:\r\n\ Please report this client's User-Agent and any other available details \ at https://github.com/abonander/multipart/issues/56", field_headers); diff --git a/multipart/src/server/save.rs b/multipart/src/server/save.rs index 822394a21..11c1b63cc 100644 --- a/multipart/src/server/save.rs +++ b/multipart/src/server/save.rs @@ -8,13 +8,15 @@ use mime::Mime; +pub use server::buf_redux::BufReader; + pub use tempdir::TempDir; use std::collections::HashMap; use std::io::prelude::*; use std::fs::{self, File, OpenOptions}; use std::path::{Path, PathBuf}; -use std::{env, io, mem}; +use std::{env, io, mem, usize}; use server::field::{FieldHeaders, MultipartField, MultipartData, ReadEntry, ReadEntryResult}; use server::ArcStr; @@ -36,6 +38,15 @@ macro_rules! try_start ( ) ); +macro_rules! try_full ( + ($try:expr) => { + match $try { + SaveResult::Full(full) => full, + other => return other, + } + } +); + #[derive(Clone, Copy, Debug, Eq, PartialEq)] enum TextPolicy { /// Attempt to read a text field as text, falling back to binary on error @@ -158,7 +169,7 @@ impl SaveBuilder { Self { text_policy: TextPolicy::Ignore, ..self } } - fn fork(&self, savable: M_) -> Self { + fn fork(&self, savable: M_) -> SaveBuilder { // this actually forking works Self { savable, .. *self } } @@ -298,59 +309,77 @@ impl SaveBuilder where M: ReadEntry { /// Save API for individual files. impl<'m, M: 'm> SaveBuilder<&'m mut MultipartData> where MultipartData: BufRead { - /// Save to a file with a random alphanumeric name in the OS temporary directory. - /// - /// Does not use user input to create the path. + /// Save the field data, potentially using a file with a random name in the + /// OS temporary directory. /// /// See `with_path()` for more details. - pub fn temp(&mut self) -> FileSaveResult { + pub fn temp(&mut self) -> FieldSaveResult { let path = env::temp_dir().join(rand_filename()); self.with_path(path) } - /// Save to a file with the given name in the OS temporary directory. + /// Save the field data, potentially using a file with the given name in + /// the OS temporary directory. /// /// See `with_path()` for more details. - /// - - pub fn with_filename(&mut self, filename: &str) -> FileSaveResult { + pub fn with_filename(&mut self, filename: &str) -> FieldSaveResult { let mut tempdir = env::temp_dir(); tempdir.set_file_name(filename); self.with_path(tempdir) } - /// Save to a file with a random alphanumeric name in the given directory. + /// Save the field data, potentially using a file with a random alphanumeric name + /// in the given directory. /// /// See `with_path()` for more details. - pub fn with_dir>(&mut self, dir: P) -> FileSaveResult { + pub fn with_dir>(&mut self, dir: P) -> FieldSaveResult { let path = dir.as_ref().join(rand_filename()); self.with_path(path) } - /// Save to a file with the given path. + /// Save the field data, potentially using a file with the given path. + /// + /// The file will not be created until the set `memory_threshold` is reached. /// /// Creates any missing directories in the path. /// Uses the contained `OpenOptions` to create the file. /// Truncates the file to the given limit, if set. - pub fn with_path>(&mut self, path: P) -> FileSaveResult { - let path = path.into(); + pub fn with_path>(&mut self, path: P) -> FieldSaveResult { + let bytes = match try_full!(self.save_mem()) { + (bytes, true) => return Full(bytes), + (bytes, false) => bytes, + }; - let saved = SavedField { - headers: self.savable.headers.clone(), - data: Data + // TODO: progressively validate UTF-8 instead + // it'd perform about the same but could give better throughput as we can do the work + // while the network buffer refills + let bytes = match self.text_policy { + TextPolicy::Try => match String::from_utf8(bytes) { + Ok(string) => return Full(SavedData::Text(string)), + Err(e) => bytes, + }, + TextPolicy::Force => match String::from_utf8(bytes) { + Ok(string) => return Full(SavedData::Text(string)), + Err(e) => return Error(io::Error::new(io::ErrorKind::InvalidData, e)), + }, + TextPolicy::Ignore => bytes, }; + let path = path.into(); + let file = match create_dir_all(&path).and_then(|_| self.open_opts.open(&path)) { Ok(file) => file, - Err(e) => return Partial(saved, e.into()) + Err(e) => return Error(e), }; - self.write_to(file).map(move |written| saved.with_size(written)) + let data = try_full!(try_write_all(&bytes).map(move |size| SavedData::File(path, size))); + + self.write_to(file).map(move |written| data.add_size(written)) } - /// Write out the file field to `dest`, truncating if a limit was set. + /// Write out the field data to `dest`, truncating if a limit was set. /// /// Returns the number of bytes copied, and whether or not the limit was reached /// (tested by `MultipartFile::fill_buf().is_empty()` so no bytes are consumed). @@ -358,19 +387,24 @@ impl<'m, M: 'm> SaveBuilder<&'m mut MultipartData> where MultipartData: Bu /// Retries on interrupts. pub fn write_to(&mut self, mut dest: W) -> SaveResult { if let Some(limit) = self.size_limit { - let copied = match try_copy_buf(self.savable.take(limit), &mut dest) { - Full(copied) => copied, - other => return other, - }; - - // If there's more data to be read, the field was truncated - match self.savable.fill_buf() { - Ok(buf) if buf.is_empty() => Full(copied), - Ok(_) => Partial(copied, PartialReason::SizeLimit), - Err(e) => Partial(copied, PartialReason::IoError(e)) - } + try_copy_limited(&mut self.savable, dest, limit) } else { - try_copy_buf(&mut self.savable, &mut dest) + try_copy_buf(self.savable, &mut dest) + } + } + + fn save_mem(&mut self) -> SaveResult<(Vec, bool), Vec> { + let mut bytes = Vec::new(); + + if self.size_limit.map_or(false, |lim| lim < self.memory_threshold) { + return self.write_to(&mut bytes).map(move |_| bytes); + } + + match try_copy_limited(self.savable, &mut bytes, self.memory_threshold) { + Full(_) => Full((bytes, true)), + Partial(_, PartialReason::SizeLimit) => Full((bytes, false)), + Partial(_, other) => Partial(bytes, other), + Error(e) => Error(e), } } } @@ -400,7 +434,7 @@ impl SavedData { match *self { Text(ref text) => Ok(DataReader::Bytes(text.as_ref())), Bytes(ref bytes) => Ok(DataReader::Bytes(bytes)), - File(ref path, _) => File::open(path), + File(ref path, _) => Ok(DataReader::File(BufReader::new(fs::File::open(path)?))), } } @@ -413,14 +447,51 @@ impl SavedData { File(_, size) => size, } } + + fn add_size(self, add: u64) -> Self { + use self::SavedData::File; + + match *self { + File(path, size) => File(path, size.saturating_add(add)), + other => other + } + } } pub enum DataReader<'a> { Bytes(&'a [u8]), - File(File), + File(BufReader), +} + +impl<'a> Read for DataReader<'a> { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + use self::DataReader::*; + + match *self { + Bytes(ref mut bytes) => bytes.read(buf), + File(ref mut file) => file.read(buf), + } + } } -imp +impl<'a> BufRead for DataReader<'a> { + fn fill_buf(&mut self) -> io::Result<&[u8]> { + use self::DataReader::*; + + match *self { + Bytes(ref mut bytes) => bytes.fill_buf(), + File(ref mut file) => file.fill_buf(), + } } + + fn consume(&mut self, amt: usize) { + use self::DataReader::*; + + match *self { + Bytes(ref mut bytes) => bytes.consume(amt), + File(ref mut file) => file.consume(amt), + } + } +} /// A result of `Multipart::save_all()`. #[derive(Debug)] @@ -576,7 +647,7 @@ impl PartialReason { /// /// May be partially saved to the filesystem if `dest` is `Some`. #[derive(Debug)] -pub struct PartialSavedField { +pub struct PartialSavedField { pub source: MultipartField, /// The partial file's entry on the filesystem, if the operation got that far. pub dest: Option, @@ -588,7 +659,7 @@ pub struct PartialSavedField { /// saved file that was in the process of being read when the error occurred, /// if applicable. #[derive(Debug)] -pub struct PartialEntries { +pub struct PartialEntries { /// The entries that were saved successfully. pub entries: Entries, /// The field that was in the process of being read. `None` if the error @@ -714,6 +785,17 @@ fn create_dir_all(path: &Path) -> io::Result<()> { } } +fn try_copy_limited(mut src: R, mut dest: W, limit: u64) -> SaveResult { + let copied = try_full!(try_copy_buf(src.by_ref().take(limit), &mut dest)); + + // If there's more data to be read, the field was truncated + match src.fill_buf() { + Ok(buf) if buf.is_empty() => Full(copied), + Ok(_) => Partial(copied, PartialReason::SizeLimit), + Err(e) => Partial(copied, PartialReason::IoError(e)) + } +} + fn try_copy_buf(mut src: R, mut dest: W) -> SaveResult { let mut total_copied = 0u64; From e99eb533d214600bdef527aebb2c5f11c0defa3d Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sun, 24 Dec 2017 18:02:22 -0800 Subject: [PATCH 353/453] finish new `save` API --- multipart/src/bin/test_multipart.rs | 4 +- multipart/src/server/field.rs | 21 +- multipart/src/server/save.rs | 383 +++++++++++++++++----------- 3 files changed, 255 insertions(+), 153 deletions(-) diff --git a/multipart/src/bin/test_multipart.rs b/multipart/src/bin/test_multipart.rs index b3c0119ce..d8dbaa956 100644 --- a/multipart/src/bin/test_multipart.rs +++ b/multipart/src/bin/test_multipart.rs @@ -51,7 +51,7 @@ fn main() { let mut multipart = Multipart::with_body(reader, boundary); while let Some(field) = multipart.read_entry().unwrap() { - println!("Read field: {:?}", field.name); + println!("Read field: {:?}", field.headers.name); } println!("All entries read!"); @@ -73,4 +73,4 @@ impl Read for RandomReader { self.inner.read(&mut buf[..len]) } -} \ No newline at end of file +} diff --git a/multipart/src/server/field.rs b/multipart/src/server/field.rs index 6f8c7ed7c..56b9d718a 100644 --- a/multipart/src/server/field.rs +++ b/multipart/src/server/field.rs @@ -119,7 +119,7 @@ impl FieldHeaders { fn parse(headers: &[StrHeader]) -> Result { let cont_disp = ContentDisp::parse(headers)?.ok_or(ParseHeaderError::MissingContentDisposition)?; Ok(FieldHeaders { - name: cont_disp.name, + name: cont_disp.field_name.into(), filename: cont_disp.filename, content_type: parse_content_type(headers)?, }) @@ -254,22 +254,29 @@ pub struct MultipartData { inner: Option, } +const DATA_INNER_ERR: &'static str = "MultipartFile::inner taken and not replaced; this is likely \ + caused by a logic error in `multipart` or by resuming after \ + a previously caught panic.\nPlease open an issue with the \ + relevant backtrace and debug logs at \ + https://github.com/abonander/multipart"; + impl MultipartData where M: ReadEntry { /// Get a builder type which can save the field with or without a size limit. pub fn save(&mut self) -> SaveBuilder<&mut Self> { SaveBuilder::new(self) } - fn inner_mut(&mut self) -> &mut M { - self.inner.as_mut().expect("MultipartFile::inner taken!") + /// Take the inner `Multipart` or `&mut Multipart` + pub fn into_inner(self) -> M { + self.inner.expect(DATA_INNER_ERR) } - fn take_inner(&mut self) -> M { - self.inner.take().expect("MultipartFile::inner already taken!") + fn inner_mut(&mut self) -> &mut M { + self.inner.as_mut().expect(DATA_INNER_ERR) } - fn into_inner(self) -> M { - self.inner.expect("MultipartFile::inner taken!") + fn take_inner(&mut self) -> M { + self.inner.take().expect(DATA_INNER_ERR) } fn give_inner(&mut self, inner: M) { diff --git a/multipart/src/server/save.rs b/multipart/src/server/save.rs index 11c1b63cc..c0681ade7 100644 --- a/multipart/src/server/save.rs +++ b/multipart/src/server/save.rs @@ -16,12 +16,14 @@ use std::collections::HashMap; use std::io::prelude::*; use std::fs::{self, File, OpenOptions}; use std::path::{Path, PathBuf}; -use std::{env, io, mem, usize}; +use std::{cmp, env, io, mem, str}; use server::field::{FieldHeaders, MultipartField, MultipartData, ReadEntry, ReadEntryResult}; use server::ArcStr; use self::SaveResult::*; +use self::TextPolicy::*; +use self::PartialReason::*; const RANDOM_FILENAME_LEN: usize = 12; @@ -33,7 +35,7 @@ macro_rules! try_start ( ($try:expr) => ( match $try { Ok(val) => val, - Err(e) => return SaveResult::Error(e), + Err(e) => return Error(e), } ) ); @@ -41,12 +43,22 @@ macro_rules! try_start ( macro_rules! try_full ( ($try:expr) => { match $try { - SaveResult::Full(full) => full, + Full(full) => full, other => return other, } } ); +macro_rules! try_partial ( + ($try:expr) => { + match $try { + Full(full) => return Full(full.into()), + Partial(partial, reason) => (partial, reason), + Error(e) => return Error(e), + } + } +); + #[derive(Clone, Copy, Debug, Eq, PartialEq)] enum TextPolicy { /// Attempt to read a text field as text, falling back to binary on error @@ -58,7 +70,7 @@ enum TextPolicy { } // 8 MiB, reasonable? -const DEFAULT_MEMORY_THRESHOLD: usize = 8 * 1024 * 1024; +const DEFAULT_MEMORY_THRESHOLD: u64 = 8 * 1024 * 1024; /// A builder for saving a file or files to the local filesystem. /// @@ -98,10 +110,11 @@ pub struct SaveBuilder { open_opts: OpenOptions, size_limit: Option, count_limit: Option, - memory_threshold: usize, + memory_threshold: u64, text_policy: TextPolicy, } +/// Common methods for whole requests as well as individual fields. impl SaveBuilder { /// Implementation detail but not problematic to have accessible. #[doc(hidden)] @@ -110,8 +123,8 @@ impl SaveBuilder { open_opts.write(true).create_new(true); SaveBuilder { - savable: savable, - open_opts: open_opts, + savable, + open_opts, size_limit: None, count_limit: None, memory_threshold: DEFAULT_MEMORY_THRESHOLD, @@ -141,17 +154,17 @@ impl SaveBuilder { /// it to disk. /// /// If `0`, forces fields to save directly to the filesystem. - /// If `usize::MAX`, effectively forces fields to always save to memory. - /// - /// (RFC: usize::MAX` is technically reachable on 32-bit systems, should we test for capacity - /// overflow and switch to disk then? What about if/when `Vec::try_reserve()` becomes a thing?) - pub fn memory_threshold(self, memory_threshold: usize) -> Self { + /// If `u64::MAX`, effectively forces fields to always save to memory. + pub fn memory_threshold(self, memory_threshold: u64) -> Self { Self { memory_threshold, ..self } } /// When encountering a field that is apparently text, try to read it to a string or fall /// back to binary otherwise. /// + /// If set for an individual field (`SaveBuilder<&mut MultipartData<_>>`), will + /// always attempt to decode text regardless of the field's `Content-Type`. + /// /// Has no effect once `memory_threshold` has been reached. pub fn try_text(self) -> Self { Self { text_policy: TextPolicy::Try, ..self } @@ -159,6 +172,9 @@ impl SaveBuilder { /// When encountering a field that is apparently text, read it to a string or return an error. /// + /// If set for an individual field (`SaveBuilder<&mut MultipartData<_>>`), will + /// always attempt to decode text regardless of the field's `Content-Type`. + /// /// (RFC: should this continue to validate UTF-8 when writing to the filesystem?) pub fn force_text(self) -> Self { Self { text_policy: TextPolicy::Force, ..self} @@ -168,11 +184,6 @@ impl SaveBuilder { pub fn ignore_text(self) -> Self { Self { text_policy: TextPolicy::Ignore, ..self } } - - fn fork(&self, savable: M_) -> SaveBuilder { - // this actually forking works - Self { savable, .. *self } - } } /// Save API for whole multipart requests. @@ -231,84 +242,83 @@ impl SaveBuilder where M: ReadEntry { /// Commence the save operation using the existing `Entries` instance. /// /// May be used to resume a saving operation after handling an error. + /// + /// If `count_limit` is set, only reads that many fields before returning an error; + /// re-running this method will read another `count_limit` fields before returning another + /// error. + /// + /// Note that `PartialReason::CountLimit` will still be returned if the number of fields over- + /// flows `u32`, but this would be an extremely degenerate case. pub fn with_entries(mut self, mut entries: Entries) -> EntriesSaveResult { - let mut count = 0; + let SaveBuilder { + savable, open_opts, count_limit, size_limit, + memory_threshold, text_policy + } = self; + + let mut res = ReadEntry::read_entry(savable); + + let save_field = |field: &mut MultipartField, entries: &Entries| { + let text_policy = if field.is_text() { text_policy } else { Ignore }; + + let mut saver = SaveBuilder { + savable: &mut field.data, open_opts: open_opts.clone(), + count_limit, size_limit, memory_threshold, text_policy + }; - loop { - let field = match ReadEntry::read_entry(self.savable) { + if let Some(dir) = entries.save_dir_path() { + saver.with_dir(dir) + } else { + saver.temp() + } + }; + + for _ in 0 .. self.count_limit.unwrap_or(::std::u32::MAX) { + let mut field: MultipartField = match res { ReadEntryResult::Entry(field) => field, - ReadEntryResult::End(_) => break, + ReadEntryResult::End(_) => return Full(entries), // normal exit point ReadEntryResult::Error(_, e) => return Partial ( PartialEntries { - entries: entries, - partial_file: None, + entries, + partial: None, }, e.into(), ) }; - match field.data { - MultipartData::File(mut file) => { - match self.count_limit { - Some(limit) if count >= limit => return Partial ( - PartialEntries { - entries: entries, - partial_file: Some(PartialSavedField { - field_name: field.name, - source: file, - dest: None, - }) - }, - PartialReason::CountLimit, - ), - _ => (), - } - - count += 1; - - match file.save().size_limit(self.size_limit).with_dir(&entries.save_dir) { - Full(saved_file) => { - self.savable = file.take_inner(); - entries.mut_files_for(field.name).push(saved_file); - }, - Partial(partial, reason) => return Partial( - PartialEntries { - entries: entries, - partial_file: Some(PartialSavedField { - field_name: field.name, - source: file, - dest: Some(partial) - }) - }, - reason - ), - Error(e) => return Partial( - PartialEntries { - entries: entries, - partial_file: Some(PartialSavedField { - field_name: field.name, - source: file, - dest: None, - }), - }, - e.into(), - ), - } + let (dest, reason) = match save_field(&mut field, &entries) { + Full(saved) => { + entries.push_field(field.headers, saved); + res = ReadEntry::read_entry(field.data.into_inner()); + continue; }, - MultipartData::Text(mut text) => { - self.savable = text.take_inner(); - entries.fields.push((field.name, text.text)); + Partial(saved, reason) => (Some(saved), reason), + Error(error) => (None, PartialReason::IoError(error)), + }; + + return Partial( + PartialEntries { + entries, + partial: Some(PartialSavedField { + source: field, + dest, + }), }, - } + reason + ); } - SaveResult::Full(entries) + Partial( + PartialEntries { + entries, + partial: None, + }, + PartialReason::CountLimit + ) } } -/// Save API for individual files. +/// Save API for individual fields. impl<'m, M: 'm> SaveBuilder<&'m mut MultipartData> where MultipartData: BufRead { - /// Save the field data, potentially using a file with a random name in the /// OS temporary directory. /// @@ -341,39 +351,42 @@ impl<'m, M: 'm> SaveBuilder<&'m mut MultipartData> where MultipartData: Bu /// Save the field data, potentially using a file with the given path. /// /// The file will not be created until the set `memory_threshold` is reached. + /// If `size_limit` is set and less than or equal to `memory_threshold`, + /// then the file will never be created. /// /// Creates any missing directories in the path. /// Uses the contained `OpenOptions` to create the file. /// Truncates the file to the given limit, if set. pub fn with_path>(&mut self, path: P) -> FieldSaveResult { - let bytes = match try_full!(self.save_mem()) { - (bytes, true) => return Full(bytes), - (bytes, false) => bytes, + let bytes = if self.text_policy != Ignore { + let (text, reason) = try_partial!(self.save_text()); + match reason { + SizeLimit if !self.cmp_size_limit(text.len()) => text.into_bytes(), + Utf8Error(_) if self.text_policy != Force => text.into_bytes(), + other => return Partial(text.into(), other), + } + } else { + Vec::new() }; - // TODO: progressively validate UTF-8 instead - // it'd perform about the same but could give better throughput as we can do the work - // while the network buffer refills - let bytes = match self.text_policy { - TextPolicy::Try => match String::from_utf8(bytes) { - Ok(string) => return Full(SavedData::Text(string)), - Err(e) => bytes, - }, - TextPolicy::Force => match String::from_utf8(bytes) { - Ok(string) => return Full(SavedData::Text(string)), - Err(e) => return Error(io::Error::new(io::ErrorKind::InvalidData, e)), - }, - TextPolicy::Ignore => bytes, - }; + let (bytes, reason) = try_partial!(self.save_mem(bytes)); + + match reason { + SizeLimit if !self.cmp_size_limit(bytes.len()) => (), + other => return Partial(bytes.into(), other) + } let path = path.into(); - let file = match create_dir_all(&path).and_then(|_| self.open_opts.open(&path)) { + let mut file = match create_dir_all(&path).and_then(|_| self.open_opts.open(&path)) { Ok(file) => file, Err(e) => return Error(e), }; - let data = try_full!(try_write_all(&bytes).map(move |size| SavedData::File(path, size))); + let data = try_full!( + try_write_all(&bytes, &mut file) + .map(move |size| SavedData::File(path, size as u64)) + ); self.write_to(file).map(move |written| data.add_size(written)) } @@ -387,47 +400,80 @@ impl<'m, M: 'm> SaveBuilder<&'m mut MultipartData> where MultipartData: Bu /// Retries on interrupts. pub fn write_to(&mut self, mut dest: W) -> SaveResult { if let Some(limit) = self.size_limit { - try_copy_limited(&mut self.savable, dest, limit) + try_copy_limited(&mut self.savable, |buf| try_write_all(buf, &mut dest), limit) } else { - try_copy_buf(self.savable, &mut dest) + try_read_buf(&mut self.savable, |buf| try_write_all(buf, &mut dest)) } } - fn save_mem(&mut self) -> SaveResult<(Vec, bool), Vec> { - let mut bytes = Vec::new(); - - if self.size_limit.map_or(false, |lim| lim < self.memory_threshold) { - return self.write_to(&mut bytes).map(move |_| bytes); + fn save_mem(&mut self, mut bytes: Vec) -> SaveResult, Vec> { + let pre_read = Some(bytes.len() as u64); + match self.read_mem(|buf| { bytes.extend_from_slice(buf); Full(buf.len()) }, pre_read) { + Full(_) => Full(bytes), + Partial(_, reason) => Partial(bytes, reason), + Error(e) => if !bytes.is_empty() { Partial(bytes, e.into()) } + else { Error(e) } } - match try_copy_limited(self.savable, &mut bytes, self.memory_threshold) { - Full(_) => Full((bytes, true)), - Partial(_, PartialReason::SizeLimit) => Full((bytes, false)), - Partial(_, other) => Partial(bytes, other), + } + + fn save_text(&mut self) -> SaveResult { + let mut string = String::new(); + + // incrementally validate UTF-8 to do as much work as possible during network activity + let res = self.read_mem(|buf| { + match str::from_utf8(buf) { + Ok(s) => { string.push_str(s); Full(buf.len()) }, + // buffer should always be bigger + Err(e) => if buf.len() < 4 { return Partial(0, e.into())} else { + string.push_str(str::from_utf8(&buf[..e.valid_up_to()]).unwrap()); + Full(e.valid_up_to()) + } + } + }, None); + + match res { + Full(_) => Full(string), + Partial(_, reason) => Partial(string, reason), Error(e) => Error(e), } } + + fn read_mem SaveResult>(&mut self, with_buf: Wb, pre_read: Option) -> SaveResult { + let limit = cmp::min(self.size_limit.unwrap_or(self.memory_threshold), self.memory_threshold) + .saturating_sub(pre_read.unwrap_or(0)); + try_copy_limited(&mut self.savable, with_buf, limit) + } + + fn cmp_size_limit(&self, size: usize) -> bool { + self.size_limit.map_or(false, |limit| size as u64 >= limit) + } } -/// A saved field (to memory or filesystem) from a multipart request. +/// A field that has been saved (to memory or disk) from a multipart request. #[derive(Debug)] pub struct SavedField { /// The headers of the field that was saved. pub headers: FieldHeaders, - /// The data of the field which may reside in-memory or on the filesystem. + /// The data of the field which may reside in memory or on disk. pub data: SavedData, } +/// A saved field's data container (in memory or on disk) #[derive(Debug)] pub enum SavedData { - /// Data in the form of a Rust string. + /// Validated UTF-8 text data. Text(String), + /// Binary data. Bytes(Vec), /// A path to a file on the filesystem and its size as written by `multipart`. File(PathBuf, u64), } impl SavedData { + /// Get an adapter for this data which implements `Read`. + /// + /// If the data is in a file, the file is opened in read-only mode. pub fn readable(&self) -> io::Result { use self::SavedData::*; @@ -438,6 +484,10 @@ impl SavedData { } } + /// Get the size of the data, in memory or on disk. + /// + /// #### Note + /// The size on disk may not match the size of the file if it is externally modified. pub fn size(&self) -> u64 { use self::SavedData::*; @@ -448,18 +498,43 @@ impl SavedData { } } + /// Returns `true` if the data is known to be in memory (`Text | Bytes`) + pub fn is_memory(&self) -> bool { + use self::SavedData::*; + + match *self { + Text(_) | Bytes(_) => true, + File(_, _) => false, + } + } + fn add_size(self, add: u64) -> Self { use self::SavedData::File; - match *self { + match self { File(path, size) => File(path, size.saturating_add(add)), other => other } } } +impl From for SavedData { + fn from(s: String) -> Self { + SavedData::Text(s) + } +} + +impl From> for SavedData { + fn from(b: Vec) -> Self { + SavedData::Bytes(b) + } +} + +/// A `Read` (and `BufRead`) adapter for `SavedData` pub enum DataReader<'a> { + /// In-memory data source (`SavedData::Bytes | Text`) Bytes(&'a [u8]), + /// On-disk data source (`SavedData::File`) File(BufReader), } @@ -481,7 +556,8 @@ impl<'a> BufRead for DataReader<'a> { match *self { Bytes(ref mut bytes) => bytes.fill_buf(), File(ref mut file) => file.fill_buf(), - } } + } + } fn consume(&mut self, amt: usize) { use self::DataReader::*; @@ -493,14 +569,17 @@ impl<'a> BufRead for DataReader<'a> { } } -/// A result of `Multipart::save_all()`. +/// A result of `Multipart::save()`. #[derive(Debug)] pub struct Entries { /// The fields of the multipart request, mapped by field name -> value. /// + /// A field name may have multiple actual fields associated with it, but the most + /// common case is a single field. + /// /// Each vector is guaranteed not to be empty unless externally modified. pub fields: HashMap>, - + /// The directory that the entries in `fields` were saved into. pub save_dir: Option, } @@ -508,17 +587,22 @@ impl Entries { fn new(save_dir: SaveDir) -> Self { Entries { fields: HashMap::new(), - save_dir: save_dir, + save_dir: Some(save_dir), } } - /// Returns `true` if both `fields` and `files` are empty, `false` otherwise. + /// Returns `true` if `fields` is empty, `false` otherwise. pub fn is_empty(&self) -> bool { - self.fields.is_empty() && self.files.is_empty() + self.fields.is_empty() } - fn mut_files_for(&mut self, field: String) -> &mut Vec { - self.files.entry(field).or_insert_with(Vec::new) + fn save_dir_path(&self) -> Option<&Path> { + self.save_dir.as_ref().map(SaveDir::as_path) + } + + fn push_field(&mut self, headers: FieldHeaders, data: SavedData) { + self.fields.entry(headers.name.clone()) + .or_insert(Vec::new()).push(SavedField { headers, data }) } } @@ -619,11 +703,19 @@ pub enum PartialReason { SizeLimit, /// An error occurred during the operation. IoError(io::Error), + /// An error returned from validating a field as UTF-8 due to `TextPolicy::Force` + Utf8Error(str::Utf8Error), } impl From for PartialReason { fn from(e: io::Error) -> Self { - PartialReason::IoError(e) + IoError(e) + } +} + +impl From for PartialReason { + fn from(e: str::Utf8Error) -> Self { + Utf8Error(e) } } @@ -648,9 +740,12 @@ impl PartialReason { /// May be partially saved to the filesystem if `dest` is `Some`. #[derive(Debug)] pub struct PartialSavedField { + /// The field that was being read. + /// + /// May be partially read if `dest` is `Some`. pub source: MultipartField, - /// The partial file's entry on the filesystem, if the operation got that far. - pub dest: Option, + /// The data from the saving operation, if it got that far. + pub dest: Option, } /// The partial result type for `Multipart::save*()`. @@ -664,25 +759,25 @@ pub struct PartialEntries { pub entries: Entries, /// The field that was in the process of being read. `None` if the error /// occurred between entries. - pub partial_field: Option>, + pub partial: Option>, } -/// Discards `partial_file` -impl Into for PartialEntries { +/// Discards `partial` +impl Into for PartialEntries { fn into(self) -> Entries { self.entries } } -impl PartialEntries { - /// If `partial_file` is present and contains a `SavedFile` then just +impl PartialEntries { + /// If `partial` is present and contains a `SavedFile` then just /// add it to the `Entries` instance and return it. /// /// Otherwise, returns `self.entries` pub fn keep_partial(mut self) -> Entries { - if let Some(partial_file) = self.partial_file { - if let Some(saved_file) = partial_file.dest { - self.entries.mut_files_for(partial_file.field_name).push(saved_file); + if let Some(partial) = self.partial { + if let Some(saved) = partial.dest { + self.entries.push_field(partial.source.headers, saved); } } @@ -703,7 +798,7 @@ pub enum SaveResult { } /// Shorthand result for methods that return `Entries` -pub type EntriesSaveResult = SaveResult>; +pub type EntriesSaveResult = SaveResult>; /// Shorthand result for methods that return `FieldData`s. /// @@ -712,7 +807,7 @@ pub type EntriesSaveResult = SaveResult>; /// saves some headache with the borrow-checker. pub type FieldSaveResult = SaveResult; -impl EntriesSaveResult { +impl EntriesSaveResult { /// Take the `Entries` from `self`, if applicable, and discarding /// the error, if any. pub fn into_entries(self) -> Option { @@ -743,7 +838,7 @@ impl SaveResult where P: Into { pub fn into_opt_both(self) -> (Option, Option) { match self { Full(full) => (Some(full), None), - Partial(partial, PartialReason::IoError(e)) => (Some(partial.into()), Some(e)), + Partial(partial, IoError(e)) => (Some(partial.into()), Some(e)), Partial(partial, _) => (Some(partial.into()), None), Error(error) => (None, Some(error)), } @@ -785,18 +880,18 @@ fn create_dir_all(path: &Path) -> io::Result<()> { } } -fn try_copy_limited(mut src: R, mut dest: W, limit: u64) -> SaveResult { - let copied = try_full!(try_copy_buf(src.by_ref().take(limit), &mut dest)); +fn try_copy_limited SaveResult>(mut src: R, mut with_buf: Wb, limit: u64) -> SaveResult { + let mut copied = 0u64; + try_read_buf(src, |buf| { + let new_copied = copied.saturating_add(buf.len() as u64); + if new_copied > limit { return Partial(0, PartialReason::SizeLimit) } + copied = new_copied; - // If there's more data to be read, the field was truncated - match src.fill_buf() { - Ok(buf) if buf.is_empty() => Full(copied), - Ok(_) => Partial(copied, PartialReason::SizeLimit), - Err(e) => Partial(copied, PartialReason::IoError(e)) - } + with_buf(buf) + }) } -fn try_copy_buf(mut src: R, mut dest: W) -> SaveResult { +fn try_read_buf SaveResult>(mut src: R, mut with_buf: Wb) -> SaveResult { let mut total_copied = 0u64; macro_rules! try_here ( @@ -814,7 +909,7 @@ fn try_copy_buf(mut src: R, mut dest: W) -> SaveResult(mut src: R, mut dest: W) -> SaveResult(mut buf: &[u8], mut dest: W) -> SaveResult where W: Write { +fn try_write_all(mut buf: &[u8], mut dest: W) -> SaveResult { let mut total_copied = 0; macro_rules! try_here ( From f364656afcffaf2d59814cbc8704aa8fab61e11f Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sun, 24 Dec 2017 18:39:20 -0800 Subject: [PATCH 354/453] documentation and API tweaks --- multipart/src/server/save.rs | 118 +++++++++++++++++++++-------------- 1 file changed, 70 insertions(+), 48 deletions(-) diff --git a/multipart/src/server/save.rs b/multipart/src/server/save.rs index c0681ade7..f44594683 100644 --- a/multipart/src/server/save.rs +++ b/multipart/src/server/save.rs @@ -16,7 +16,7 @@ use std::collections::HashMap; use std::io::prelude::*; use std::fs::{self, File, OpenOptions}; use std::path::{Path, PathBuf}; -use std::{cmp, env, io, mem, str}; +use std::{cmp, env, io, mem, str, u32, u64}; use server::field::{FieldHeaders, MultipartField, MultipartData, ReadEntry, ReadEntryResult}; use server::ArcStr; @@ -69,9 +69,6 @@ enum TextPolicy { Ignore } -// 8 MiB, reasonable? -const DEFAULT_MEMORY_THRESHOLD: u64 = 8 * 1024 * 1024; - /// A builder for saving a file or files to the local filesystem. /// /// ### `OpenOptions` @@ -108,8 +105,8 @@ const DEFAULT_MEMORY_THRESHOLD: u64 = 8 * 1024 * 1024; pub struct SaveBuilder { savable: S, open_opts: OpenOptions, - size_limit: Option, - count_limit: Option, + size_limit: u64, + count_limit: u32, memory_threshold: u64, text_policy: TextPolicy, } @@ -125,18 +122,19 @@ impl SaveBuilder { SaveBuilder { savable, open_opts, - size_limit: None, - count_limit: None, - memory_threshold: DEFAULT_MEMORY_THRESHOLD, + size_limit: u64::MAX, + count_limit: u32::MAX, + // 8 MiB, reasonable? + memory_threshold: 8 * 1024 * 1024, text_policy: TextPolicy::Try, } } /// Set the maximum number of bytes to write out *per file*. /// - /// Can be `u64` or `Option`. If `None`, clears the limit. + /// Can be `u64` or `Option`. If `None` or `u64::MAX`, clears the limit. pub fn size_limit>>(mut self, limit: L) -> Self { - self.size_limit = limit.into(); + self.size_limit = limit.into().unwrap_or(u64::MAX); self } @@ -188,30 +186,34 @@ impl SaveBuilder { /// Save API for whole multipart requests. impl SaveBuilder where M: ReadEntry { - /// Set the maximum number of files to write out. + /// Set the maximum number of fields to process. /// - /// Can be `u32` or `Option`. If `None`, clears the limit. + /// Can be `u32` or `Option`. If `None` or `u32::MAX`, clears the limit. pub fn count_limit>>(mut self, count_limit: L) -> Self { - self.count_limit = count_limit.into(); + self.count_limit = count_limit.into().unwrap_or(u32::MAX); self } - /// Save the file fields in the request to a new temporary directory prefixed with + /// Save all fields in the request using a new temporary directory prefixed with /// `multipart-rs` in the OS temporary directory. /// /// For more options, create a `TempDir` yourself and pass it to `with_temp_dir()` instead. /// + /// See `with_entries()` for more info. + /// /// ### Note: Temporary /// See `SaveDir` for more info (the type of `Entries::save_dir`). pub fn temp(self) -> EntriesSaveResult { self.temp_with_prefix("multipart-rs") } - /// Save the file fields in the request to a new temporary directory with the given string + /// Save all fields in the request using a new temporary directory with the given string /// as a prefix in the OS temporary directory. /// /// For more options, create a `TempDir` yourself and pass it to `with_temp_dir()` instead. /// + /// See `with_entries()` for more info. + /// /// ### Note: Temporary /// See `SaveDir` for more info (the type of `Entries::save_dir`). pub fn temp_with_prefix(self, prefix: &str) -> EntriesSaveResult { @@ -221,7 +223,9 @@ impl SaveBuilder where M: ReadEntry { } } - /// Save the file fields to the given `TempDir`. + /// Save all fields in the request using the given `TempDir`. + /// + /// See `with_entries()` for more info. /// /// The `TempDir` is returned in the result under `Entries::save_dir`. pub fn with_temp_dir(self, tempdir: TempDir) -> EntriesSaveResult { @@ -231,6 +235,8 @@ impl SaveBuilder where M: ReadEntry { /// Save the file fields in the request to a new permanent directory with the given path. /// /// Any nonexistent directories in the path will be created. + /// + /// See `with_entries()` for more info. pub fn with_dir>(self, dir: P) -> EntriesSaveResult { let dir = dir.into(); @@ -243,12 +249,11 @@ impl SaveBuilder where M: ReadEntry { /// /// May be used to resume a saving operation after handling an error. /// - /// If `count_limit` is set, only reads that many fields before returning an error; - /// re-running this method will read another `count_limit` fields before returning another - /// error. + /// If `count_limit` is set, only reads that many fields before returning an error. + /// If you wish to resume from `PartialReason::CountLimit`, simply remove some entries. /// - /// Note that `PartialReason::CountLimit` will still be returned if the number of fields over- - /// flows `u32`, but this would be an extremely degenerate case. + /// Note that `PartialReason::CountLimit` will still be returned if the number of fields + /// reaches `u32::MAX`, but this would be an extremely degenerate case. pub fn with_entries(mut self, mut entries: Entries) -> EntriesSaveResult { let SaveBuilder { savable, open_opts, count_limit, size_limit, @@ -257,6 +262,8 @@ impl SaveBuilder where M: ReadEntry { let mut res = ReadEntry::read_entry(savable); + let _ = entries.recount_fields(); + let save_field = |field: &mut MultipartField, entries: &Entries| { let text_policy = if field.is_text() { text_policy } else { Ignore }; @@ -265,14 +272,10 @@ impl SaveBuilder where M: ReadEntry { count_limit, size_limit, memory_threshold, text_policy }; - if let Some(dir) = entries.save_dir_path() { - saver.with_dir(dir) - } else { - saver.temp() - } + saver.with_dir(entries.save_dir.as_path()) }; - for _ in 0 .. self.count_limit.unwrap_or(::std::u32::MAX) { + while entries.fields_count < count_limit { let mut field: MultipartField = match res { ReadEntryResult::Entry(field) => field, ReadEntryResult::End(_) => return Full(entries), // normal exit point @@ -350,13 +353,13 @@ impl<'m, M: 'm> SaveBuilder<&'m mut MultipartData> where MultipartData: Bu /// Save the field data, potentially using a file with the given path. /// - /// The file will not be created until the set `memory_threshold` is reached. - /// If `size_limit` is set and less than or equal to `memory_threshold`, - /// then the file will never be created. - /// - /// Creates any missing directories in the path. + /// Creates any missing directories in the path (RFC: skip this step?). /// Uses the contained `OpenOptions` to create the file. - /// Truncates the file to the given limit, if set. + /// Truncates the file to the given `size_limit`, if set. + /// + /// The no directories or files will be created until the set `memory_threshold` is reached. + /// If `size_limit` is set and less than or equal to `memory_threshold`, + /// then the disk will never be touched. pub fn with_path>(&mut self, path: P) -> FieldSaveResult { let bytes = if self.text_policy != Ignore { let (text, reason) = try_partial!(self.save_text()); @@ -399,15 +402,15 @@ impl<'m, M: 'm> SaveBuilder<&'m mut MultipartData> where MultipartData: Bu /// /// Retries on interrupts. pub fn write_to(&mut self, mut dest: W) -> SaveResult { - if let Some(limit) = self.size_limit { - try_copy_limited(&mut self.savable, |buf| try_write_all(buf, &mut dest), limit) + if self.size_limit < u64::MAX { + try_copy_limited(&mut self.savable, |buf| try_write_all(buf, &mut dest), self.size_limit) } else { try_read_buf(&mut self.savable, |buf| try_write_all(buf, &mut dest)) } } fn save_mem(&mut self, mut bytes: Vec) -> SaveResult, Vec> { - let pre_read = Some(bytes.len() as u64); + let pre_read = bytes.len() as u64; match self.read_mem(|buf| { bytes.extend_from_slice(buf); Full(buf.len()) }, pre_read) { Full(_) => Full(bytes), Partial(_, reason) => Partial(bytes, reason), @@ -430,7 +433,7 @@ impl<'m, M: 'm> SaveBuilder<&'m mut MultipartData> where MultipartData: Bu Full(e.valid_up_to()) } } - }, None); + }, 0); match res { Full(_) => Full(string), @@ -439,14 +442,14 @@ impl<'m, M: 'm> SaveBuilder<&'m mut MultipartData> where MultipartData: Bu } } - fn read_mem SaveResult>(&mut self, with_buf: Wb, pre_read: Option) -> SaveResult { - let limit = cmp::min(self.size_limit.unwrap_or(self.memory_threshold), self.memory_threshold) - .saturating_sub(pre_read.unwrap_or(0)); + fn read_mem SaveResult>(&mut self, with_buf: Wb, pre_read: u64) -> SaveResult { + let limit = cmp::min(self.size_limit, self.memory_threshold) + .saturating_sub(pre_read); try_copy_limited(&mut self.savable, with_buf, limit) } fn cmp_size_limit(&self, size: usize) -> bool { - self.size_limit.map_or(false, |limit| size as u64 >= limit) + size as u64 >= self.size_limit } } @@ -580,14 +583,17 @@ pub struct Entries { /// Each vector is guaranteed not to be empty unless externally modified. pub fields: HashMap>, /// The directory that the entries in `fields` were saved into. - pub save_dir: Option, + pub save_dir: SaveDir, + fields_count: u32, } impl Entries { - fn new(save_dir: SaveDir) -> Self { + /// Create a new `Entries` with the given `SaveDir` + pub fn new(save_dir: SaveDir) -> Self { Entries { fields: HashMap::new(), - save_dir: Some(save_dir), + save_dir, + fields_count: 0, } } @@ -596,13 +602,29 @@ impl Entries { self.fields.is_empty() } - fn save_dir_path(&self) -> Option<&Path> { - self.save_dir.as_ref().map(SaveDir::as_path) + /// The number of actual fields contained within this `Entries`. + /// + /// Effectively `self.fields.values().map(Vec::len).sum()` but maintained separately. + /// + /// ## Note + /// This will be incorrect if `fields` is modified externally. Call `recount_fields()` + /// to get the correct count. + pub fn fields_count(&self) -> u32 { + self.fields_count + } + + /// Sum the number of fields in this `Entries` and then return the updated value. + pub fn recount_fields(&mut self) -> u32 { + let fields_count = self.fields.values().map(Vec::len).sum(); + // saturating cast + self.fields_count = cmp::min(u32::MAX as usize, fields_count) as u32; + self.fields_count } fn push_field(&mut self, headers: FieldHeaders, data: SavedData) { self.fields.entry(headers.name.clone()) - .or_insert(Vec::new()).push(SavedField { headers, data }) + .or_insert(Vec::new()).push(SavedField { headers, data }); + self.fields_count = self.fields_count.saturating_add(1); } } From 7c9628f5304c5301028ac7124f16fb1d7a90a02f Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sun, 24 Dec 2017 18:53:48 -0800 Subject: [PATCH 355/453] Document memory threshold and text policy on `SaveBuilder` --- multipart/src/server/save.rs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/multipart/src/server/save.rs b/multipart/src/server/save.rs index f44594683..32a5475d2 100644 --- a/multipart/src/server/save.rs +++ b/multipart/src/server/save.rs @@ -90,6 +90,24 @@ enum TextPolicy { /// takes either `u32` or `Option`. This only has an effect when using /// `SaveBuilder<[&mut] Multipart>`. /// +/// ### Memory Threshold and Text Policy +/// By default, small fields (a few megabytes or smaller) will be read directly to memory +/// without creating a file. This behavior is controlled by the `memory_threshold()` setter. +/// +/// If a field appears to contain text data (its content-type is `text/*` or it doesn't declare +/// one), `SaveBuilder` can read it to a string instead of saving the raw bytes as long as it falls +/// below the set `memory_threshold`. +/// +/// By default, the behavior is to attempt to validate the data as UTF-8, falling back to saving +/// just the bytes if the validation fails at any point. You can restore/ensure this behavior +/// with the `try_text()` modifier. +/// +/// Alternately, you can use the `force_text()` modifier to have the save operation return an error +/// when UTF-8 decoding fails, though this only holds true while the size is below +/// `memory_threshold`. The `ignore_text()` modifier turns off UTF-8 validation altogether. +/// +/// UTF-8 validation is performed incrementally to hopefully maximize throughput. +/// /// ### Warning: Do **not** trust user input! /// It is a serious security risk to create files or directories with paths based on user input. /// A malicious user could craft a path which can be used to overwrite important files, such as From cddf6663f841cf5dbf57ff7c33d8acd22200d6c3 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sun, 24 Dec 2017 19:14:21 -0800 Subject: [PATCH 356/453] Document memory threshold and text policy on `SaveBuilder` --- multipart/src/server/save.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/multipart/src/server/save.rs b/multipart/src/server/save.rs index 32a5475d2..ab2c6b6e0 100644 --- a/multipart/src/server/save.rs +++ b/multipart/src/server/save.rs @@ -102,11 +102,15 @@ enum TextPolicy { /// just the bytes if the validation fails at any point. You can restore/ensure this behavior /// with the `try_text()` modifier. /// -/// Alternately, you can use the `force_text()` modifier to have the save operation return an error -/// when UTF-8 decoding fails, though this only holds true while the size is below +/// Alternatively, you can use the `force_text()` modifier to make the save operation return +/// an error when UTF-8 decoding fails, though this only holds true while the size is below /// `memory_threshold`. The `ignore_text()` modifier turns off UTF-8 validation altogether. /// -/// UTF-8 validation is performed incrementally to hopefully maximize throughput. +/// UTF-8 validation is performed incrementally (after every `BufRead::fill_buf()` call) +/// to hopefully maximize throughput, instead of blocking while the field is read to completion +/// and performing validation over the entire result at the end. (RFC: this could be a lot of +/// unnecessary work if most fields end up being written to the filesystem, however, but this +/// can be turned off with `ignore_text()` if it fits the use-case.) /// /// ### Warning: Do **not** trust user input! /// It is a serious security risk to create files or directories with paths based on user input. From 40977719f0de0cc9e99c74cf259324bc4ab2a93e Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sun, 24 Dec 2017 19:43:51 -0800 Subject: [PATCH 357/453] remove testing `multipart-nickel` from Travis --- multipart/.travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/multipart/.travis.yml b/multipart/.travis.yml index 86def6adc..38fb49257 100644 --- a/multipart/.travis.yml +++ b/multipart/.travis.yml @@ -24,4 +24,3 @@ script: - cargo build --verbose; - cargo test --verbose; - cargo doc --verbose; - - (cd nickel && cargo test -v) From a3e59a14ece687ba9149e8e6542cc20f21b0e4c7 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sun, 24 Dec 2017 19:53:40 -0800 Subject: [PATCH 358/453] fix Hyper server integration --- multipart/Cargo.toml | 2 +- multipart/src/server/hyper.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 7d78bbe86..4fdf6487d 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -40,7 +40,7 @@ env_logger = "0.3" [features] client = [] -default = ["server"] +default = ["hyper", "server"] server = ["buf_redux", "httparse", "safemem", "twoway"] mock = [] nightly = [] diff --git a/multipart/src/server/hyper.rs b/multipart/src/server/hyper.rs index 4299b144b..e1697395a 100644 --- a/multipart/src/server/hyper.rs +++ b/multipart/src/server/hyper.rs @@ -16,7 +16,7 @@ use hyper::server::{Handler, Request, Response}; pub use hyper::server::Request as HyperRequest; -use mime::{Mime, TopLevel, SubLevel, Attr, Value}; +use hyper::mime::{Mime, TopLevel, SubLevel, Attr, Value}; use super::{Multipart, HttpRequest}; From 7cc9d4a2089097945147b69cbcf902f09a10b0b7 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sun, 24 Dec 2017 23:04:43 -0800 Subject: [PATCH 359/453] fix Iron integration --- multipart/Cargo.toml | 2 +- multipart/src/server/iron.rs | 18 ++++++++---------- multipart/src/server/mod.rs | 2 +- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 4fdf6487d..f8bf84eca 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -40,7 +40,7 @@ env_logger = "0.3" [features] client = [] -default = ["hyper", "server"] +default = ["iron", "hyper", "server"] server = ["buf_redux", "httparse", "safemem", "twoway"] mock = [] nightly = [] diff --git a/multipart/src/server/iron.rs b/multipart/src/server/iron.rs index ece7aec42..43d050812 100644 --- a/multipart/src/server/iron.rs +++ b/multipart/src/server/iron.rs @@ -12,7 +12,7 @@ use iron::{BeforeMiddleware, IronError, IronResult}; use std::path::PathBuf; use std::{error, fmt, io}; -use super::{HttpRequest, Multipart}; +use super::{FieldHeaders, HttpRequest, Multipart}; use super::save::{Entries, PartialReason, TempDir}; use super::save::SaveResult::*; @@ -142,14 +142,14 @@ impl Intercept { .count_limit(self.file_count_limit) .with_temp_dir(tempdir) { Full(entries) => Ok(Some(entries)), + Partial(_, PartialReason::Utf8Error(_)) => unreachable!(), Partial(_, PartialReason::IoError(err)) => Err(io_to_iron(err, "Error midway through request")), Partial(_, PartialReason::CountLimit) => Err(FileCountLimitError(self.file_count_limit).into()), Partial(partial, PartialReason::SizeLimit) => { - let partial_file = partial.partial_file.expect(EXPECT_PARTIAL_FILE); + let partial = partial.partial.expect(EXPECT_PARTIAL_FILE); Err( FileSizeLimitError { - field: partial_file.field_name, - filename: partial_file.source.filename, + field: partial.source.headers, }.into() ) }, @@ -229,9 +229,7 @@ pub enum LimitBehavior { #[derive(Debug)] pub struct FileSizeLimitError { /// The field where the error occurred. - pub field: String, - /// The filename of the oversize file, if it was provided. - pub filename: Option, + pub field: FieldHeaders, } impl error::Error for FileSizeLimitError { @@ -242,9 +240,9 @@ impl error::Error for FileSizeLimitError { impl fmt::Display for FileSizeLimitError { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - match self.filename { - Some(ref filename) => write!(f, "File size limit reached for field \"{}\" (filename: \"{}\")", self.field, filename), - None => write!(f, "File size limit reached for field \"{}\" (no filename)", self.field), + match self.field.filename { + Some(ref filename) => write!(f, "File size limit reached for field \"{}\" (filename: \"{}\")", self.field.name, filename), + None => write!(f, "File size limit reached for field \"{}\" (no filename)", self.field.name), } } } diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index 2f550f938..5d2a7e527 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -27,7 +27,7 @@ use self::boundary::BoundaryReader; use self::field::PrivReadEntry; -pub use self::field::{MultipartField, MultipartData, ReadEntry, ReadEntryResult}; +pub use self::field::{FieldHeaders, MultipartField, MultipartData, ReadEntry, ReadEntryResult}; use self::save::SaveBuilder; From a7e50b1e8035a4e8f93f938d2a855ad0b5933f22 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sun, 21 Jan 2018 21:05:51 -0800 Subject: [PATCH 360/453] wip fixing examples --- multipart/Cargo.toml | 2 +- multipart/examples/hyper_server.rs | 2 +- multipart/examples/nickel.rs | 33 +++++------------------------- multipart/examples/tiny_http.rs | 22 -------------------- multipart/src/client/hyper.rs | 2 +- multipart/src/local_test.rs | 8 ++++---- multipart/src/mock.rs | 27 ++++++++++++++++++++++++ multipart/src/server/save.rs | 25 ++++++++++++++++++++++ 8 files changed, 64 insertions(+), 57 deletions(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index f8bf84eca..3a13b7ae4 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -40,7 +40,7 @@ env_logger = "0.3" [features] client = [] -default = ["iron", "hyper", "server"] +default = ["client", "hyper", "iron", "nickel", "server", "tiny_http"] server = ["buf_redux", "httparse", "safemem", "twoway"] mock = [] nightly = [] diff --git a/multipart/examples/hyper_server.rs b/multipart/examples/hyper_server.rs index d141c69fe..c9fc059a1 100644 --- a/multipart/examples/hyper_server.rs +++ b/multipart/examples/hyper_server.rs @@ -46,7 +46,7 @@ fn process_entries<'a>(entries: Entries) -> io::Result<()> { println!("Field {:?}: {:?}", name, field); } - for (name, files) in entries.files { + for (name, files) in &entries.fields { println!("Field {:?} has {} files:", name, files.len()); for file in files { diff --git a/multipart/examples/nickel.rs b/multipart/examples/nickel.rs index d05aca846..dfba184a6 100644 --- a/multipart/examples/nickel.rs +++ b/multipart/examples/nickel.rs @@ -2,11 +2,12 @@ extern crate multipart; extern crate nickel; use std::fs::File; -use std::io::Read; +use std::io::{self, Read}; use nickel::{HttpRouter, MiddlewareResult, Nickel, Request, Response}; -use multipart_nickel::MultipartBody; -use multipart_nickel::multipart_server::{Entries, SaveResult}; +use multipart::nickel::MultipartBody; +use multipart::server::{Entries, SaveResult}; +use multipart::mock::StdoutTee; fn handle_multipart<'mw>(req: &mut Request, mut res: Response<'mw>) -> MiddlewareResult<'mw> { match req.multipart_body() { @@ -36,32 +37,8 @@ fn handle_multipart<'mw>(req: &mut Request, mut res: Response<'mw>) -> Middlewar /// Processes saved entries from multipart request. /// Returns an OK response or an error. fn process_entries<'mw>(res: Response<'mw>, entries: Entries) -> MiddlewareResult<'mw> { - for (name, field) in entries.fields { - println!("Field {:?}: {:?}", name, field); - } - - for (name, files) in entries.files { - println!("Field {:?} has {} files:", name, files.len()); + if let Err(e) = entries.write_debug(StdoutTee::new(res.start()?)) { - for saved_file in files { - match File::open(&saved_file.path) { - Ok(mut file) => { - let mut contents = String::new(); - if let Err(e) = file.read_to_string(&mut contents) { - println!("Could not read file {:?}. Error: {:?}", saved_file.filename, e); - return res.error(nickel::status::StatusCode::BadRequest, "The uploaded file was not readable") - } - - println!("File {:?} ({:?}):", saved_file.filename, saved_file.content_type); - println!("{}", contents); - file - } - Err(e) => { - println!("Could open file {:?}. Error: {:?}", saved_file.filename, e); - return res.error(nickel::status::StatusCode::BadRequest, "The uploaded file was not readable") - } - }; - } } res.send("Ok") diff --git a/multipart/examples/tiny_http.rs b/multipart/examples/tiny_http.rs index 101927732..e9b0c5a3c 100644 --- a/multipart/examples/tiny_http.rs +++ b/multipart/examples/tiny_http.rs @@ -52,33 +52,11 @@ fn process_request<'a, 'b>(request: &'a mut Request) -> io::Result(entries: Entries) -> io::Result> { - for (name, field) in entries.fields { - println!("Field {:?}: {:?}", name, field); - } - for (name, files) in entries.files { - println!("Field {:?} has {} files:", name, files.len()); - - for file in files { - print_file(&file)?; - } - } Ok(build_response(200, "Multipart data is received!")) } -fn print_file(saved_file: &SavedField) -> io::Result<()> { - let mut file = File::open(&saved_file.path)?; - - let mut contents = String::new(); - file.read_to_string(&mut contents)?; - - println!("File {:?} ({:?}):", saved_file.filename, saved_file.content_type); - println!("{}", contents); - - Ok(()) -} - /// A utility function to build responses using only two arguments fn build_response(status_code: u16, response: &str) -> Response<&[u8]> { let bytes = response.as_bytes(); diff --git a/multipart/src/client/hyper.rs b/multipart/src/client/hyper.rs index 6172f96f7..675cba6d1 100644 --- a/multipart/src/client/hyper.rs +++ b/multipart/src/client/hyper.rs @@ -20,7 +20,7 @@ use hyper::net::{Fresh, Streaming}; use hyper::Error as HyperError; -use mime::{Mime, TopLevel, SubLevel, Attr, Value}; +use hyper::mime::{Mime, TopLevel, SubLevel, Attr, Value}; use super::{HttpRequest, HttpStream}; diff --git a/multipart/src/local_test.rs b/multipart/src/local_test.rs index bcecfa680..9ab0b3746 100644 --- a/multipart/src/local_test.rs +++ b/multipart/src/local_test.rs @@ -8,7 +8,7 @@ use mock::{ClientRequest, HttpBuffer}; use server::{MultipartField, MultipartData, ReadEntry}; -use mime::Mime; +use mime::{self, Mime}; use rand::{self, Rng}; @@ -380,8 +380,8 @@ fn rand_mime() -> Mime { rand::thread_rng().choose(&[ // TODO: fill this out, preferably with variants that may be hard to parse // i.e. containing hyphens, mainly - mime!(Application/OctetStream), - mime!(Text/Plain), - mime!(Image/Png), + mime::APPLICATION_OCTET_STREAM, + mime::TEXT_PLAIN, + mime::IMAGE_PNG, ]).unwrap().clone() } diff --git a/multipart/src/mock.rs b/multipart/src/mock.rs index f9cb3d336..017dd8bc3 100644 --- a/multipart/src/mock.rs +++ b/multipart/src/mock.rs @@ -173,3 +173,30 @@ impl<'a> ::server::HttpRequest for ServerRequest<'a> { self } } + +/// A `Write` adapter that duplicates all data written to the inner writer as well as stdout. +pub struct StdoutTee { + inner: W, + stdout: io::Stdout, +} + +impl StdoutTee { + /// Constructor + pub fn new(inner: W) -> Self { + Self { + inner, stdout: io::stdout(), + } + } +} + +impl Write for StdoutTee { + fn write(&mut self, buf: &[u8]) -> io::Result { + inner.write(buf)?; + stdout.write(buf) + } + + fn flush(&mut self) -> io::Result<()> { + inner.flush(); + stdout.flush() + } +} diff --git a/multipart/src/server/save.rs b/multipart/src/server/save.rs index ab2c6b6e0..8604f6cf5 100644 --- a/multipart/src/server/save.rs +++ b/multipart/src/server/save.rs @@ -648,6 +648,31 @@ impl Entries { .or_insert(Vec::new()).push(SavedField { headers, data }); self.fields_count = self.fields_count.saturating_add(1); } + + /// Print all fields and their contents to stdout. Mostly for testing purposes. + pub fn print_debug(&self) -> io::Result<()> { + let stdout = io::stdout(); + write_debug(stdout.lock()) + } + + /// Write all fields and their contents to the given output. Mostly for testing purposes. + pub fn write_debug(&self, mut writer: W) -> io::Result<()> { + for (name, field) in &self.fields { + write!(writer, "Field {:?}: {:?}", name, field); + } + + for (name, fields) in &entries.fields { + writeln!(writer, "Field {:?} has {} entries:", name, fields.len()); + + for (idx, field) in fields.iter().enumerate() { + let mut data = saved_field.data.readable()?; + + writeln!(writer, "Field {:?}({}) ({:?}):", + saved_field.filename, idx, saved_field.content_type); + io::copy(&mut data, &mut writer); + } + } + } } /// The save directory for `Entries`. May be temporary (delete-on-drop) or permanent. From 54f2ab14c3fe25ca2ef6c09f3a8d501237bb1012 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sat, 27 Jan 2018 16:34:46 -0800 Subject: [PATCH 361/453] wip cargo-check passes --- multipart/examples/nickel.rs | 23 +++++++++++++++++------ multipart/src/mock.rs | 20 ++++++++++---------- multipart/src/server/save.rs | 26 ++++++++++++-------------- 3 files changed, 39 insertions(+), 30 deletions(-) diff --git a/multipart/examples/nickel.rs b/multipart/examples/nickel.rs index dfba184a6..d08269600 100644 --- a/multipart/examples/nickel.rs +++ b/multipart/examples/nickel.rs @@ -3,7 +3,8 @@ extern crate nickel; use std::fs::File; use std::io::{self, Read}; -use nickel::{HttpRouter, MiddlewareResult, Nickel, Request, Response}; +use nickel::{Action, Fresh, HttpRouter, MiddlewareResult, Nickel, NickelError, Request, Response}; +use nickel::status::StatusCode; use multipart::nickel::MultipartBody; use multipart::server::{Entries, SaveResult}; @@ -22,13 +23,13 @@ fn handle_multipart<'mw>(req: &mut Request, mut res: Response<'mw>) -> Middlewar SaveResult::Error(e) => { println!("There are errors in multipart POSTing ... {:?}", e); - res.set(nickel::status::StatusCode::InternalServerError); + res.set(StatusCode::InternalServerError); return res.send(format!("Server could not handle multipart POST! {:?}", e)); }, } } None => { - res.set(nickel::status::StatusCode::BadRequest); + res.set(StatusCode::BadRequest); return res.send("Request seems not was a multipart request") } } @@ -37,11 +38,21 @@ fn handle_multipart<'mw>(req: &mut Request, mut res: Response<'mw>) -> Middlewar /// Processes saved entries from multipart request. /// Returns an OK response or an error. fn process_entries<'mw>(res: Response<'mw>, entries: Entries) -> MiddlewareResult<'mw> { - if let Err(e) = entries.write_debug(StdoutTee::new(res.start()?)) { - + let stdout = io::stdout(); + let mut res = res.start()?; + if let Err(e) = entries.write_debug(StdoutTee::new(&mut res, stdout.lock())) { + writeln!(res, "Error while reading entries: {}", e); + return Err(NickelError::new( + res, StatusCode::InternalServerError, format!("Error while reading entries: {}", e) + )); } - res.send("Ok") + if let Err(e) = res.end() { + // I don't like this, see https://github.com/nickel-org/nickel.rs/issues/427 + unsafe { + Err(NickelError::without_response("error flushing response")) + } + } } fn main() { diff --git a/multipart/src/mock.rs b/multipart/src/mock.rs index 017dd8bc3..cee48b369 100644 --- a/multipart/src/mock.rs +++ b/multipart/src/mock.rs @@ -175,28 +175,28 @@ impl<'a> ::server::HttpRequest for ServerRequest<'a> { } /// A `Write` adapter that duplicates all data written to the inner writer as well as stdout. -pub struct StdoutTee { +pub struct StdoutTee<'s, W> { inner: W, - stdout: io::Stdout, + stdout: io::StdoutLock<'s>, } -impl StdoutTee { +impl<'s, W> StdoutTee<'s, W> { /// Constructor - pub fn new(inner: W) -> Self { + pub fn new(inner: W, stdout: &'s io::Stdout) -> Self { Self { - inner, stdout: io::stdout(), + inner, stdout: stdout.lock(), } } } -impl Write for StdoutTee { +impl<'s, W: Write> Write for StdoutTee<'s, W> { fn write(&mut self, buf: &[u8]) -> io::Result { - inner.write(buf)?; - stdout.write(buf) + self.inner.write(buf)?; + self.stdout.write(buf) } fn flush(&mut self) -> io::Result<()> { - inner.flush(); - stdout.flush() + self.inner.flush(); + self.stdout.flush() } } diff --git a/multipart/src/server/save.rs b/multipart/src/server/save.rs index 8604f6cf5..253fa4c84 100644 --- a/multipart/src/server/save.rs +++ b/multipart/src/server/save.rs @@ -652,26 +652,24 @@ impl Entries { /// Print all fields and their contents to stdout. Mostly for testing purposes. pub fn print_debug(&self) -> io::Result<()> { let stdout = io::stdout(); - write_debug(stdout.lock()) + let stdout_lock = stdout.lock(); + self.write_debug(stdout_lock) } /// Write all fields and their contents to the given output. Mostly for testing purposes. pub fn write_debug(&self, mut writer: W) -> io::Result<()> { - for (name, field) in &self.fields { - write!(writer, "Field {:?}: {:?}", name, field); - } - - for (name, fields) in &entries.fields { - writeln!(writer, "Field {:?} has {} entries:", name, fields.len()); - - for (idx, field) in fields.iter().enumerate() { - let mut data = saved_field.data.readable()?; - - writeln!(writer, "Field {:?}({}) ({:?}):", - saved_field.filename, idx, saved_field.content_type); - io::copy(&mut data, &mut writer); + for (name, entries) in &self.fields { + writeln!(writer, "Field {:?} has {} entries:", name, entries.len())?; + + for (idx, field) in entries.iter().enumerate() { + let mut data = field.data.readable()?; + let headers = &field.headers; + writeln!(writer, "{}: {:?} ({:?}):", idx, headers.filename, headers.content_type)?; + io::copy(&mut data, &mut writer)?; } } + + Ok(()) } } From 8305c8177877a056c26fe0c19a730f046907fa24 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Tue, 30 Jan 2018 22:00:49 -0800 Subject: [PATCH 362/453] more WIP fixes --- multipart/Cargo.toml | 2 +- multipart/examples/hyper_server.rs | 35 +++--------- multipart/examples/nickel.rs | 15 ++--- multipart/src/local_test.rs | 92 ++++++++++++------------------ multipart/src/server/nickel.rs | 1 - 5 files changed, 51 insertions(+), 94 deletions(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 3a13b7ae4..30e8c90ae 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -40,7 +40,7 @@ env_logger = "0.3" [features] client = [] -default = ["client", "hyper", "iron", "nickel", "server", "tiny_http"] +default = ["client", "hyper", "iron", "mock", "nickel", "server", "tiny_http"] server = ["buf_redux", "httparse", "safemem", "twoway"] mock = [] nightly = [] diff --git a/multipart/examples/hyper_server.rs b/multipart/examples/hyper_server.rs index c9fc059a1..bc89e2913 100644 --- a/multipart/examples/hyper_server.rs +++ b/multipart/examples/hyper_server.rs @@ -8,6 +8,7 @@ use hyper::status::StatusCode; use hyper::server::response::Response as HyperResponse; use multipart::server::hyper::{Switch, MultipartHandler, HyperRequest}; use multipart::server::{Multipart, Entries, SaveResult, SavedField}; +use multipart::mock::StdoutTee; struct NonMultipart; impl Handler for NonMultipart { @@ -21,10 +22,10 @@ struct EchoMultipart; impl MultipartHandler for EchoMultipart { fn handle_multipart(&self, mut multipart: Multipart, mut res: HyperResponse) { let processing = match multipart.save().temp() { - SaveResult::Full(entries) => process_entries(entries), + SaveResult::Full(entries) => process_entries(&mut res, entries), SaveResult::Partial(entries, error) => { println!("Errors saving multipart:\n{:?}", error); - process_entries(entries.into()) + process_entries(&mut res, entries.into()) } SaveResult::Error(error) => { println!("Errors saving multipart:\n{:?}", error); @@ -41,32 +42,10 @@ impl MultipartHandler for EchoMultipart { } } -fn process_entries<'a>(entries: Entries) -> io::Result<()> { - for (name, field) in entries.fields { - println!("Field {:?}: {:?}", name, field); - } - - for (name, files) in &entries.fields { - println!("Field {:?} has {} files:", name, files.len()); - - for file in files { - print_file(&file)?; - } - } - - Ok(()) -} - -fn print_file(saved_file: &SavedField) -> io::Result<()> { - let mut file = File::open(&saved_file.path)?; - - let mut contents = String::new(); - file.read_to_string(&mut contents)?; - - println!("File {:?} ({:?}):", saved_file.filename, saved_file.content_type); - println!("{}", contents); - - Ok(()) +fn process_entries(res: &mut HyperResponse, entries: Entries) -> io::Result<()> { + let mut res = res.start(); + let stdout = io::stdout(); + entries.write_debug(StdoutTee::new(&mut res, &stdout)) } fn main() { diff --git a/multipart/examples/nickel.rs b/multipart/examples/nickel.rs index d08269600..02262f69f 100644 --- a/multipart/examples/nickel.rs +++ b/multipart/examples/nickel.rs @@ -2,11 +2,11 @@ extern crate multipart; extern crate nickel; use std::fs::File; -use std::io::{self, Read}; -use nickel::{Action, Fresh, HttpRouter, MiddlewareResult, Nickel, NickelError, Request, Response}; +use std::io::{self, Read, Write}; +use nickel::{Action, HttpRouter, MiddlewareResult, Nickel, NickelError, Request, Response}; use nickel::status::StatusCode; -use multipart::nickel::MultipartBody; +use multipart::server::nickel::MultipartBody; use multipart::server::{Entries, SaveResult}; use multipart::mock::StdoutTee; @@ -40,19 +40,14 @@ fn handle_multipart<'mw>(req: &mut Request, mut res: Response<'mw>) -> Middlewar fn process_entries<'mw>(res: Response<'mw>, entries: Entries) -> MiddlewareResult<'mw> { let stdout = io::stdout(); let mut res = res.start()?; - if let Err(e) = entries.write_debug(StdoutTee::new(&mut res, stdout.lock())) { + if let Err(e) = entries.write_debug(StdoutTee::new(&mut res, &stdout)) { writeln!(res, "Error while reading entries: {}", e); return Err(NickelError::new( res, StatusCode::InternalServerError, format!("Error while reading entries: {}", e) )); } - if let Err(e) = res.end() { - // I don't like this, see https://github.com/nickel-org/nickel.rs/issues/427 - unsafe { - Err(NickelError::without_response("error flushing response")) - } - } + Ok(Action::Halt(res)) } fn main() { diff --git a/multipart/src/local_test.rs b/multipart/src/local_test.rs index 9ab0b3746..0420219e0 100644 --- a/multipart/src/local_test.rs +++ b/multipart/src/local_test.rs @@ -41,74 +41,53 @@ macro_rules! expect_fmt ( #[derive(Debug)] struct TestFields { - texts: HashMap, - files: HashMap>, + fields: HashMap>, } impl TestFields { fn gen() -> Self { TestFields { - texts: collect_rand(|| (gen_string(), gen_string())), - files: collect_rand(|| (gen_string(), FileEntry::gen_many())), + fields: collect_rand(|| (gen_string(), FieldEntry::gen_many())), } } - fn check_field(&mut self, field: &mut MultipartField) { - match field.data { - MultipartData::Text(ref text) => { - let test_text = expect_fmt!(self.texts.remove(&field.name), - "Got text field that wasn't in original dataset: {:?} : {:?} ", - field.name, text.text - ); - - assert!( - text.text == test_text, - "Unexpected data for field {:?}: Expected {:?}, got {:?}", - field.name, test_text, text.text - ); - }, - MultipartData::File(ref mut file) => { - let mut bytes = Vec::with_capacity(MAX_LEN); - file.read_to_end(&mut bytes).unwrap(); - - let curr_file = FileEntry { - content_type: file.content_type.clone(), - filename: file.filename.take(), - data: PrintHex(bytes), - }; - - let files_empty = { - let mut files = expect_fmt!(self.files.get_mut(&field.name), - "Got file field that wasn't in original dataset: {:?} : {:?}", - field.name, curr_file); - - assert!(files.remove(&curr_file), "Unexpected data for file field {:?}: {:?}", - field.name, curr_file); - - files.is_empty() - }; - - if files_empty { - let _ = self.files.remove(&field.name); - } - }, - } + fn check_field(&mut self, field: MultipartField) { + let field_entries = expect_fmt!(self.fields.remove(&*field.headers.name), + "Got field that wasn't in original dataset: {:?}", + field.headers); + + let test_entry = FieldEntry::from_field(field); + + assert!(field_entries.contains(&test_entry), + "Got field entry that wasn't in original dataset: {:?}\nEntries: {:?}", + test_entry, field_entries + ); } fn assert_is_empty(&self) { - assert!(self.texts.is_empty(), "Text fields were not exhausted! Text fields: {:?}", self.texts); - assert!(self.files.is_empty(), "File fields were not exhausted! File fields: {:?}", self.files); + assert!(self.fields.is_empty(), "Fields were not exhausted! {:?}", self.fields); } } #[derive(Debug, Hash, PartialEq, Eq)] -struct FileEntry { +struct FieldEntry { content_type: Mime, filename: Option, data: PrintHex, } -impl FileEntry { +impl FieldEntry { + fn from_field(mut field: MultipartField) -> FieldEntry { + let mut data = Vec::new(); + field.data.read_to_end(&mut data).expect("failed reading field"); + + FieldEntry { + content_type: field.headers.content_type.unwrap_or(mime::APPLICATION_OCTET_STREAM), + filename: field.headers.filename, + data: PrintHex(data), + } + } + fn gen_many() -> HashSet { collect_rand(Self::gen) } @@ -119,10 +98,15 @@ impl FileEntry { false => None, }; - FileEntry { + let data = PrintHex(match gen_bool() { + true => gen_string().into_bytes(), + false => gen_bytes(), + }); + + FieldEntry { content_type: rand_mime(), - filename: filename, - data: PrintHex(gen_bytes()) + filename, + data, } } @@ -274,7 +258,7 @@ fn test_client(test_fields: &TestFields) -> HttpBuffer { let request = ClientRequest::default(); - let mut test_files = test_fields.files.iter(); + let mut test_files = test_fields.fields.iter(); let mut multipart = Multipart::from_request(request).unwrap(); @@ -306,7 +290,7 @@ fn test_client_lazy(test_fields: &TestFields) -> HttpBuffer { let mut multipart = Multipart::new(); - let mut test_files = test_fields.files.iter(); + let mut test_files = test_fields.fields.iter(); for (name, text) in &test_fields.texts { if let Some((file_name, files)) = test_files.next() { @@ -351,7 +335,7 @@ fn test_server(buf: HttpBuffer, fields: &mut TestFields) { .unwrap_or_else(|_| panic!("Buffer should be multipart!")); while let Some(mut field) = multipart.read_entry_mut().unwrap_opt() { - fields.check_field(&mut field); + fields.check_field(field); } } diff --git a/multipart/src/server/nickel.rs b/multipart/src/server/nickel.rs index fa9f1fca2..4fd8fd142 100644 --- a/multipart/src/server/nickel.rs +++ b/multipart/src/server/nickel.rs @@ -38,7 +38,6 @@ impl<'mw, 'server, D: 'mw> MultipartBody<'mw, 'server> for NickelRequest<'mw, 's } } - impl<'r, 'mw: 'r, 'server: 'mw, D: 'mw> AsRef<&'r mut NickelRequest<'mw, 'server, D>> for Maybe<'r, 'mw, 'server, D> { fn as_ref(&self) -> &&'r mut NickelRequest<'mw, 'server, D> { &self.0 From 1a22c230b849ab9627406f1d7a45fd7c55a613e4 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Thu, 1 Feb 2018 04:05:38 -0800 Subject: [PATCH 363/453] Entries::push_field(): dedup field names by reusing hashmap keys in `FieldHeaders` --- multipart/src/server/save.rs | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/multipart/src/server/save.rs b/multipart/src/server/save.rs index 253fa4c84..5040c092f 100644 --- a/multipart/src/server/save.rs +++ b/multipart/src/server/save.rs @@ -643,9 +643,18 @@ impl Entries { self.fields_count } - fn push_field(&mut self, headers: FieldHeaders, data: SavedData) { - self.fields.entry(headers.name.clone()) - .or_insert(Vec::new()).push(SavedField { headers, data }); + fn push_field(&mut self, mut headers: FieldHeaders, data: SavedData) { + use std::collections::hash_map::Entry::*; + + match self.fields.entry(headers.name.clone()) { + Vacant(vacant) => { vacant.insert(vec![SavedField { headers, data }]); }, + Occupied(occupied) => { + // dedup the field name by reusing the key's `Arc` + headers.name = occupied.key().clone(); + occupied.into_mut().push({ SavedField { headers, data }}); + }, + } + self.fields_count = self.fields_count.saturating_add(1); } From dfaf1c0e0364ab0da0149d9d3a0b9dd3b73fec83 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Thu, 1 Feb 2018 04:19:04 -0800 Subject: [PATCH 364/453] clarify support for synchronous APIs only --- multipart/README.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/multipart/README.md b/multipart/README.md index 34b3ab0d6..57f8c28fe 100644 --- a/multipart/README.md +++ b/multipart/README.md @@ -2,7 +2,8 @@ Client- and server-side abstractions for HTTP file uploads (POST requests with `Content-Type: multipart/form-data`). -Supports several different HTTP crates. +Supports several different (**sync**hronous API) HTTP crates. +**Async**hronous API support will be provided by [multipart-async]. Minimum supported Rust version: 1.17.0 @@ -15,6 +16,8 @@ Example files demonstrating how to use `multipart` with these crates are availab ### [Hyper](http://hyper.rs) via the `hyper` feature (enabled by default). +**Note: Hyper 0.9, 0.10 (synchronous API) only**; support for asynchronous APIs will be provided by [multipart-async]. + Client integration includes support for regular `hyper::client::Request` objects via `multipart::client::Multipart`, as well as integration with the new `hyper::Client` API via `multipart::client::lazy::Multipart` (new in 0.5). @@ -50,3 +53,5 @@ at your option. Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. + +[multipart-async]: https://github.com/abonander/multipart-async From 7639981f50a988ef454c89456da035abc56d1576 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sat, 3 Feb 2018 03:01:08 -0800 Subject: [PATCH 365/453] fix local_test --- multipart/src/local_test.rs | 169 +++++++++++++++++++++++++----------- 1 file changed, 118 insertions(+), 51 deletions(-) diff --git a/multipart/src/local_test.rs b/multipart/src/local_test.rs index 0420219e0..04150a989 100644 --- a/multipart/src/local_test.rs +++ b/multipart/src/local_test.rs @@ -6,13 +6,14 @@ // copied, modified, or distributed except according to those terms. use mock::{ClientRequest, HttpBuffer}; -use server::{MultipartField, MultipartData, ReadEntry}; +use server::{MultipartField, MultipartData, ReadEntry, FieldHeaders}; use mime::{self, Mime}; use rand::{self, Rng}; use std::collections::{HashMap, HashSet}; +use std::collections::hash_map::{Entry, OccupiedEntry}; use std::fmt; use std::io::prelude::*; use std::io::Cursor; @@ -39,56 +40,119 @@ macro_rules! expect_fmt ( ); ); +/// The error is provided as the `err` format argument +macro_rules! expect_ok_fmt ( + ($val:expr, $($args:tt)*) => ( + match $val { + Ok(val) => val, + Err(e) => panic!($($args)*, err=e), + } + ); +); + +fn get_field<'m, V>(field: &FieldHeaders, fields: &'m mut HashMap) -> Option> { + match fields.entry(field.name.to_string()) { + Entry::Occupied(occupied) => Some(occupied), + Entry::Vacant(_) => None, + } +} + #[derive(Debug)] struct TestFields { - fields: HashMap>, + texts: HashMap>, + files: HashMap>, } impl TestFields { fn gen() -> Self { TestFields { - fields: collect_rand(|| (gen_string(), FieldEntry::gen_many())), + texts: collect_rand(|| (gen_string(), collect_rand(gen_string))), + files: collect_rand(|| (gen_string(), FileEntry::gen_many())), } } - fn check_field(&mut self, field: MultipartField) { - let field_entries = expect_fmt!(self.fields.remove(&*field.headers.name), - "Got field that wasn't in original dataset: {:?}", + fn check_field(&mut self, mut field: MultipartField) -> M { + if field.is_text() { + let mut text_entries = expect_fmt!(get_field(&field.headers, &mut self.texts), + "Got text field that wasn't in original dataset: {:?}", field.headers); - let test_entry = FieldEntry::from_field(field); + let mut text = String::new(); + expect_ok_fmt!( + field.data.read_to_string(&mut text), + "error failed to read text data to string: {:?}\n{err}", field.headers + ); + + assert!( + text_entries.get_mut().remove(&text), + "Got field text data that wasn't in original data set: {:?}\n{:?}\n{:?}", + field.headers, + text, + text_entries.get(), + ); + + if text_entries.get().is_empty() { + text_entries.remove_entry(); + } + + return field.data.into_inner(); + } - assert!(field_entries.contains(&test_entry), - "Got field entry that wasn't in original dataset: {:?}\nEntries: {:?}", - test_entry, field_entries + + let mut file_entries = expect_fmt!(get_field(&field.headers, &mut self.files), + "Got file field that wasn't in original dataset: {:?}", + field.headers); + + let field_name = field.headers.name.clone(); + let (test_entry, inner) = FileEntry::from_field(field); + + assert!( + file_entries.get_mut().remove(&test_entry), + "Got field entry that wasn't in original dataset: name: {:?}\n{:?}\nEntries: {:?}", + field_name, + test_entry, + file_entries.get() ); + + if file_entries.get().is_empty() { + file_entries.remove_entry(); + } + + return inner; } fn assert_is_empty(&self) { - assert!(self.fields.is_empty(), "Fields were not exhausted! {:?}", self.fields); + assert!(self.texts.is_empty(), "Text Fields were not exhausted! {:?}", self.texts); + assert!(self.files.is_empty(), "File Fields were not exhausted! {:?}", self.files); } } #[derive(Debug, Hash, PartialEq, Eq)] -struct FieldEntry { +struct FileEntry { content_type: Mime, filename: Option, data: PrintHex, } -impl FieldEntry { - fn from_field(mut field: MultipartField) -> FieldEntry { +impl FileEntry { + fn from_field(mut field: MultipartField) -> (FileEntry, M) { let mut data = Vec::new(); - field.data.read_to_end(&mut data).expect("failed reading field"); + expect_ok_fmt!( + field.data.read_to_end(&mut data), + "Error reading file field: {:?}\n{err}", field.headers + ); - FieldEntry { - content_type: field.headers.content_type.unwrap_or(mime::APPLICATION_OCTET_STREAM), - filename: field.headers.filename, - data: PrintHex(data), - } + ( + FileEntry { + content_type: field.headers.content_type.unwrap_or(mime::APPLICATION_OCTET_STREAM), + filename: field.headers.filename, + data: PrintHex(data), + }, + field.data.into_inner() + ) } - fn gen_many() -> HashSet { + fn gen_many() -> HashSet { collect_rand(Self::gen) } @@ -103,7 +167,7 @@ impl FieldEntry { false => gen_bytes(), }); - FieldEntry { + FileEntry { content_type: rand_mime(), filename, data, @@ -115,7 +179,7 @@ impl FieldEntry { } } -#[derive(Hash, PartialEq, Eq)] +#[derive(PartialEq, Eq, Hash)] struct PrintHex(Vec); impl fmt::Debug for PrintHex { @@ -258,28 +322,30 @@ fn test_client(test_fields: &TestFields) -> HttpBuffer { let request = ClientRequest::default(); - let mut test_files = test_fields.fields.iter(); + let mut test_files = test_fields.files.iter().flat_map( + |(name, files)| files.iter().map(move |file| (name, file)) + ); + + let mut test_texts = test_fields.texts.iter().flat_map( + |(name, texts)| texts.iter().map(move |text| (name, text)) + ); let mut multipart = Multipart::from_request(request).unwrap(); // Intersperse file fields amongst text fields - for (name, text) in &test_fields.texts { - if let Some((file_name, files)) = test_files.next() { - for file in files { - multipart.write_stream(file_name, &mut &*file.data.0, file.filename(), - Some(file.content_type.clone())).unwrap(); - } + for (name, text) in test_texts { + if let Some((file_name, file)) = test_files.next() { + multipart.write_stream(file_name, &mut &*file.data.0, file.filename(), + Some(file.content_type.clone())).unwrap(); } multipart.write_text(name, text).unwrap(); } // Write remaining files - for (file_name, files) in test_files { - for file in files { - multipart.write_stream(file_name, &mut &*file.data.0, file.filename(), - Some(file.content_type.clone())).unwrap(); - } + for (file_name, file) in test_files { + multipart.write_stream(file_name, &mut &*file.data.0, file.filename(), + Some(file.content_type.clone())).unwrap(); } multipart.send().unwrap() @@ -290,24 +356,26 @@ fn test_client_lazy(test_fields: &TestFields) -> HttpBuffer { let mut multipart = Multipart::new(); - let mut test_files = test_fields.fields.iter(); + let mut test_files = test_fields.files.iter().flat_map( + |(name, files)| files.iter().map(move |file| (name, file)) + ); - for (name, text) in &test_fields.texts { - if let Some((file_name, files)) = test_files.next() { - for file in files { + let mut test_texts = test_fields.texts.iter().flat_map( + |(name, texts)| texts.iter().map(move |text| (name, text)) + ); + + for (name, text) in test_texts { + if let Some((file_name, file)) = test_files.next() { multipart.add_stream(&**file_name, Cursor::new(&file.data.0), file.filename(), Some(file.content_type.clone())); - } } multipart.add_text(&**name, &**text); } - for (file_name, files) in test_files { - for file in files { - multipart.add_stream(&**file_name, Cursor::new(&file.data.0), file.filename(), - Some(file.content_type.clone())); - } + for (file_name, file) in test_files { + multipart.add_stream(&**file_name, Cursor::new(&file.data.0), file.filename(), + Some(file.content_type.clone())); } let mut prepared = multipart.prepare().unwrap(); @@ -348,15 +416,14 @@ fn test_server_entry_api(buf: HttpBuffer, fields: &mut TestFields) { assert!(content_len == server_buf.data.len() as u64, "Supplied content_len different from actual"); } - let multipart = Multipart::from_request(server_buf) + let mut multipart = Multipart::from_request(server_buf) .unwrap_or_else(|_| panic!("Buffer should be multipart!")); - let mut entry = multipart.into_entry().expect_alt("Expected entry, got none", "Error reading entry"); - fields.check_field(&mut entry); + let entry = multipart.into_entry().expect_alt("Expected entry, got none", "Error reading entry"); + multipart = fields.check_field(entry); - while let Some(entry_) = entry.next_entry().unwrap_opt() { - entry = entry_; - fields.check_field(&mut entry); + while let Some(entry) = multipart.into_entry().unwrap_opt() { + multipart = fields.check_field(entry); } } From 14505e562444fd823ae651f145a168a403a54e24 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sun, 4 Feb 2018 15:49:33 -0800 Subject: [PATCH 366/453] fix `hyper_server`, `nickel` examples --- multipart/Cargo.toml | 4 ++-- multipart/examples/hyper_server.rs | 23 ++++++++--------------- multipart/examples/nickel.rs | 2 +- multipart/src/server/mod.rs | 7 +++++-- 4 files changed, 16 insertions(+), 20 deletions(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 30e8c90ae..0c26e1e0b 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -40,7 +40,7 @@ env_logger = "0.3" [features] client = [] -default = ["client", "hyper", "iron", "mock", "nickel", "server", "tiny_http"] +default = ["client", "hyper", "iron", "mock", "nickel", "server", "tiny_http", "use_arc_str"] server = ["buf_redux", "httparse", "safemem", "twoway"] mock = [] nightly = [] @@ -50,4 +50,4 @@ bench = [] sse4 = ["nightly", "twoway/pcmp"] all = ["client", "server", "hyper", "iron", "tiny_http", "nickel", "mock"] # switch uses of `Arc` (`From` impl only stabilized in 1.21) for `Arc` -no_arc_str = [] +use_arc_str = [] diff --git a/multipart/examples/hyper_server.rs b/multipart/examples/hyper_server.rs index bc89e2913..404a25593 100644 --- a/multipart/examples/hyper_server.rs +++ b/multipart/examples/hyper_server.rs @@ -21,31 +21,24 @@ impl Handler for NonMultipart { struct EchoMultipart; impl MultipartHandler for EchoMultipart { fn handle_multipart(&self, mut multipart: Multipart, mut res: HyperResponse) { - let processing = match multipart.save().temp() { - SaveResult::Full(entries) => process_entries(&mut res, entries), + match multipart.save().temp() { + SaveResult::Full(entries) => process_entries(res, entries).unwrap(), SaveResult::Partial(entries, error) => { println!("Errors saving multipart:\n{:?}", error); - process_entries(&mut res, entries.into()) + process_entries(res, entries.into()).unwrap(); } SaveResult::Error(error) => { println!("Errors saving multipart:\n{:?}", error); - Err(error) + res.send(format!("An error occurred {}", error).as_bytes()).unwrap(); } }; - match processing { - Ok(_) => res.send(b"All good in the hood :)\n").unwrap(), - Err(_) => { - *res.status_mut() = StatusCode::BadRequest; - res.send(b"An error occurred :(\n").unwrap(); - } - } } } -fn process_entries(res: &mut HyperResponse, entries: Entries) -> io::Result<()> { - let mut res = res.start(); - let stdout = io::stdout(); - entries.write_debug(StdoutTee::new(&mut res, &stdout)) +fn process_entries(res: HyperResponse, entries: Entries) -> io::Result<()> { + let mut res = res.start()?; + let ref stdout = io::stdout(); + entries.write_debug(StdoutTee::new(&mut res, stdout)) } fn main() { diff --git a/multipart/examples/nickel.rs b/multipart/examples/nickel.rs index 02262f69f..497f77718 100644 --- a/multipart/examples/nickel.rs +++ b/multipart/examples/nickel.rs @@ -11,7 +11,7 @@ use multipart::server::{Entries, SaveResult}; use multipart::mock::StdoutTee; fn handle_multipart<'mw>(req: &mut Request, mut res: Response<'mw>) -> MiddlewareResult<'mw> { - match req.multipart_body() { + match (*req).multipart_body() { Some(mut multipart) => { match multipart.save().temp() { SaveResult::Full(entries) => process_entries(res, entries), diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index 5d2a7e527..dd05ca325 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -36,14 +36,14 @@ pub use self::save::{Entries, SaveResult, SavedField}; use self::save::EntriesSaveResult; /// Replacement typedef for `Arc` for older Rust releases. -#[cfg(feature = "no_arc_str")] +#[cfg(not(feature = "use_arc_str"))] pub type ArcStr = Arc; /// Typedef for `Arc`. /// /// Construction of `Arc` was only stabilized in Rust 1.21, so to continue to support /// older versions, an alternate typedef of `Arc` is available under the `no_arc_str` feature. -#[cfg(not(feature = "no_arc_str"))] +#[cfg(feature = "use_arc_str")] pub type ArcStr = Arc; macro_rules! try_opt ( @@ -85,6 +85,9 @@ pub mod iron; #[cfg(feature = "tiny_http")] pub mod tiny_http; +#[cfg(feature = "nickel")] +pub mod nickel; + pub mod save; /// The server-side implementation of `multipart/form-data` requests. From ef82857f1a588456ec662bd2bd74cbc2a916eb14 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sun, 4 Feb 2018 17:29:40 -0800 Subject: [PATCH 367/453] tweaks, decouple `nickel` integration from `hyper` integration, make travis tests smarter --- multipart/.travis.yml | 19 ++++++++++++------- multipart/Cargo.toml | 19 ++++++++++++++++++- multipart/examples/nickel.rs | 3 --- multipart/src/server/mod.rs | 2 +- multipart/src/server/nickel.rs | 13 +++++++++---- multipart/src/server/save.rs | 30 +++++++++++++++++++++--------- 6 files changed, 61 insertions(+), 25 deletions(-) diff --git a/multipart/.travis.yml b/multipart/.travis.yml index 38fb49257..8ee210c95 100644 --- a/multipart/.travis.yml +++ b/multipart/.travis.yml @@ -14,13 +14,18 @@ os: # openssl is broken on OS X again # - osx env: - - RUST_LOG=multipart=info RUST_BACKTRACE=1 + global: + - RUST_LOG=multipart=info RUST_BACKTRACE=1 ARGS= + matrix: + - ARGS+=--no-default-features + # ensure nickel works without hyper + - ARGS+='--no-default-features --features "nickel"' script: - if [ ${TRAVIS_RUST_VERSION} = "nightly" ]; then - cargo build --verbose --features "nightly"; - cargo test --verbose --features "nightly"; - cargo doc --verbose --features "nightly"; + cargo build --verbose $ARGS --features "nightly"; + cargo test --verbose $ARGS --features "nightly"; + cargo doc --verbose $ARGS --features "nightly"; fi - - cargo build --verbose; - - cargo test --verbose; - - cargo doc --verbose; + - cargo build --verbose $ARGS; + - cargo test --verbose $ARGS; + - cargo doc --verbose $ARGS; diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 0c26e1e0b..1734a95c7 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -48,6 +48,23 @@ bench = [] # Use this to enable SSE4.2 instructions in boundary finding # TODO: Benchmark this sse4 = ["nightly", "twoway/pcmp"] -all = ["client", "server", "hyper", "iron", "tiny_http", "nickel", "mock"] # switch uses of `Arc` (`From` impl only stabilized in 1.21) for `Arc` use_arc_str = [] + +[examples.hyper_client] +required_features = ["hyper"] + +[examples.hyper_server] +required_features = ["hyper"] + +[examples.iron] +required_features = ["iron"] + +[examples.iron_intercept] +required_features = ["iron"] + +[examples.nickel] +required_features = ["nickel"] + +[examples.tiny_http] +required_features = ["tiny_http"] diff --git a/multipart/examples/nickel.rs b/multipart/examples/nickel.rs index 497f77718..b789c04bf 100644 --- a/multipart/examples/nickel.rs +++ b/multipart/examples/nickel.rs @@ -42,9 +42,6 @@ fn process_entries<'mw>(res: Response<'mw>, entries: Entries) -> MiddlewareResul let mut res = res.start()?; if let Err(e) = entries.write_debug(StdoutTee::new(&mut res, &stdout)) { writeln!(res, "Error while reading entries: {}", e); - return Err(NickelError::new( - res, StatusCode::InternalServerError, format!("Error while reading entries: {}", e) - )); } Ok(Action::Halt(res)) diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index dd05ca325..a97c64aef 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -11,7 +11,7 @@ //! //! See the `Multipart` struct for more info. -extern crate buf_redux; +pub extern crate buf_redux; extern crate httparse; extern crate twoway; diff --git a/multipart/src/server/nickel.rs b/multipart/src/server/nickel.rs index 4fd8fd142..0f725a3b2 100644 --- a/multipart/src/server/nickel.rs +++ b/multipart/src/server/nickel.rs @@ -1,8 +1,11 @@ //! Support for `multipart/form-data` bodies in [Nickel](https://nickel.rs). -extern crate nickel; +pub extern crate nickel; -use self::nickel::Request as NickelRequest; -use self::nickel::hyper::server::Request as HyperRequest; +use self::nickel::hyper; +use self::hyper::header::ContentType; + +pub use self::nickel::Request as NickelRequest; +pub use self::nickel::hyper::server::Request as HyperRequest; use server::{HttpRequest, Multipart}; @@ -16,7 +19,9 @@ impl<'r, 'mw: 'r, 'server: 'mw, D: 'mw> HttpRequest for Maybe<'r, 'mw, 'server, type Body = &'r mut HyperRequest<'mw, 'server>; fn multipart_boundary(&self) -> Option<&str> { - self.0.origin.multipart_boundary() + // we can't use the impl from the `hyper` module because it might be the wrong version + let cont_type = try_opt!(self.0.origin.headers.get::()); + cont_type.get_param("boundary").map(|v| v.as_str()) } fn body(self) -> Self::Body { diff --git a/multipart/src/server/save.rs b/multipart/src/server/save.rs index 5040c092f..3283c6d52 100644 --- a/multipart/src/server/save.rs +++ b/multipart/src/server/save.rs @@ -83,16 +83,23 @@ enum TextPolicy { /// `mod_open_opts()`. /// /// ### File Size and Count Limits -/// You can set a size limit for individual files with `size_limit()`, which takes either `u64` +/// You can set a size limit for individual fields with `size_limit()`, which takes either `u64` /// or `Option`. /// -/// You can also set the maximum number of files to process with `count_limit()`, which +/// You can also set the maximum number of fields to process with `count_limit()`, which /// takes either `u32` or `Option`. This only has an effect when using /// `SaveBuilder<[&mut] Multipart>`. /// +/// By default, these limits are set conservatively to limit the maximum memory and disk space +/// usage of a single request. You should set `count_limit` specifically for each request endpoint +/// based on the number of fields you're expecting (exactly to that number if you're not expecting +/// duplicate fields). +/// /// ### Memory Threshold and Text Policy -/// By default, small fields (a few megabytes or smaller) will be read directly to memory -/// without creating a file. This behavior is controlled by the `memory_threshold()` setter. +/// By default, small fields (a few kilobytes or smaller) will be read directly to memory +/// without creating a file. This behavior is controlled by the `memory_threshold()` setter. You can +/// *roughly* tune the maximum memory a single request uses by tuning +/// `count_limit * memory_threshold` /// /// If a field appears to contain text data (its content-type is `text/*` or it doesn't declare /// one), `SaveBuilder` can read it to a string instead of saving the raw bytes as long as it falls @@ -144,10 +151,13 @@ impl SaveBuilder { SaveBuilder { savable, open_opts, - size_limit: u64::MAX, - count_limit: u32::MAX, - // 8 MiB, reasonable? - memory_threshold: 8 * 1024 * 1024, + // 8 MiB, on the conservative end compared to most frameworks + size_limit: 8 * 1024 * 1024, + // Arbitrary, I have no empirical data for this + count_limit: 256, + // 10KiB, used by Apache Commons + // https://commons.apache.org/proper/commons-fileupload/apidocs/org/apache/commons/fileupload/disk/DiskFileItemFactory.html + memory_threshold: 10 * 1024, text_policy: TextPolicy::Try, } } @@ -603,6 +613,8 @@ pub struct Entries { /// common case is a single field. /// /// Each vector is guaranteed not to be empty unless externally modified. + // Even though individual fields might only have one entry, it's better to limit the + // size of a value type in `HashMap` to improve cache efficiency in lookups. pub fields: HashMap>, /// The directory that the entries in `fields` were saved into. pub save_dir: SaveDir, @@ -779,7 +791,7 @@ pub enum PartialReason { SizeLimit, /// An error occurred during the operation. IoError(io::Error), - /// An error returned from validating a field as UTF-8 due to `TextPolicy::Force` + /// An error returned from validating a field as UTF-8 due to `SaveBuilder::force_text()` Utf8Error(str::Utf8Error), } From 8d47a65e0f2bef296fb2ec32a39c7c04ea190b0f Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sun, 4 Feb 2018 17:57:36 -0800 Subject: [PATCH 368/453] boundary: ditch `fill_buf_min()`, use `buf_redux` strategies --- multipart/src/server/boundary.rs | 31 +++++++++---------------------- 1 file changed, 9 insertions(+), 22 deletions(-) diff --git a/multipart/src/server/boundary.rs b/multipart/src/server/boundary.rs index 8c1d064ea..cd37b6ba7 100644 --- a/multipart/src/server/boundary.rs +++ b/multipart/src/server/boundary.rs @@ -10,6 +10,7 @@ use ::safemem; use super::buf_redux::BufReader; +use super::buf_redux::strategy::{LessThan, AtEndLessThan}; use super::twoway; use log::LogLevel; @@ -32,7 +33,7 @@ enum State { /// A struct implementing `Read` and `BufRead` that will yield bytes until it sees a given sequence. #[derive(Debug)] pub struct BoundaryReader { - source: BufReader, + source: BufReader, boundary: Vec, search_idx: usize, state: State, @@ -45,22 +46,19 @@ impl BoundaryReader where R: Read { safemem::prepend(b"--", &mut boundary); BoundaryReader { - source: BufReader::new(reader), - boundary: boundary, + source: BufReader::with_strategies( + reader, + LessThan(boundary.len() * 2), + AtEndLessThan(boundary.len() * 2), + ), + boundary, search_idx: 0, state: Searching, } } fn read_to_boundary(&mut self) -> io::Result<&[u8]> { - // Make sure there's enough bytes in the buffer to positively identify the boundary. - let min_len = self.search_idx + (self.boundary.len() * 2); - - let buf = fill_buf_min(&mut self.source, min_len)?; - - if buf.is_empty() { - debug!("fill_buf_min returned zero-sized buf"); - } + let buf = self.source.fill_buf()?; trace!("Buf: {:?}", String::from_utf8_lossy(buf)); @@ -215,17 +213,6 @@ impl BufRead for BoundaryReader where R: Read { } } -fn fill_buf_min(buf: &mut BufReader, min: usize) -> io::Result<&[u8]> { - let mut attempts = 0; - - while buf.available() < min && attempts < min { - if buf.read_into_buf()? == 0 { break; }; - attempts += 1; - } - - Ok(buf.get_buf()) -} - #[cfg(test)] mod test { use super::BoundaryReader; From 0e81b4c2f78861d8e14130c16ced9cf240013be5 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sun, 4 Feb 2018 17:58:08 -0800 Subject: [PATCH 369/453] acknowledge key dependencies in `README` --- multipart/README.md | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/multipart/README.md b/multipart/README.md index 57f8c28fe..21a51af06 100644 --- a/multipart/README.md +++ b/multipart/README.md @@ -38,8 +38,25 @@ Provides server-side integration with `tiny_http::Request` via `multipart::serve Provides server-side integration with `&mut nickel::Request` via `multipart::server::Multipart`. -## License +## âš¡âš¡ Powered By âš¡âš¡ + +* [buf_redux ![](https://img.shields.io/crates/v/buf_redux.svg)](https://crates.io/crates/buf_redux) + +Customizable drop-in `std::io::BufReader` replacement, created to be used in this crate. +Needed because it can read more bytes into the buffer without the buffer being empty, necessary +when a boundary falls across two reads. (It was easier to author a new crate than try to get this added +to `std::io::BufReader`.) + +* [httparse ![](https://img.shields.io/crates/v/httparse.svg)](https://crates.io/crates/httparse) + +Fast, zero-copy HTTP header parsing, used to read field headers in `multipart/form-data` request bodies. + +* [twoway ![](https://img.shields.io/crates/v/twoway.svg)](https://crates.io/crates/twoway) + +Used to find boundaries in the request body. + +## License Licensed under either of From 9eb7c8bc593b618e2460a9bae9bf62fc4ea0facc Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sun, 4 Feb 2018 17:59:33 -0800 Subject: [PATCH 370/453] note SSE4.2 accelerated impl for `twoway` --- multipart/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/multipart/README.md b/multipart/README.md index 21a51af06..4c23f2ed3 100644 --- a/multipart/README.md +++ b/multipart/README.md @@ -54,7 +54,8 @@ Fast, zero-copy HTTP header parsing, used to read field headers in `multipart/fo * [twoway ![](https://img.shields.io/crates/v/twoway.svg)](https://crates.io/crates/twoway) -Used to find boundaries in the request body. +Fast string and byte-string search. Used to find boundaries in the request body. SEE 4.2 acceleration available +under the `sse42` or `twoway/pcmp` features. ## License From 2924a5ae09474f030e505808bfd585b90c06a822 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sun, 4 Feb 2018 18:00:37 -0800 Subject: [PATCH 371/453] fix typo --- multipart/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/README.md b/multipart/README.md index 4c23f2ed3..17ef46920 100644 --- a/multipart/README.md +++ b/multipart/README.md @@ -54,7 +54,7 @@ Fast, zero-copy HTTP header parsing, used to read field headers in `multipart/fo * [twoway ![](https://img.shields.io/crates/v/twoway.svg)](https://crates.io/crates/twoway) -Fast string and byte-string search. Used to find boundaries in the request body. SEE 4.2 acceleration available +Fast string and byte-string search. Used to find boundaries in the request body. SSE 4.2 acceleration available under the `sse42` or `twoway/pcmp` features. ## License From aff6376245d4f5146a33927779720ea330582e22 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sun, 4 Feb 2018 18:04:38 -0800 Subject: [PATCH 372/453] fix superscript --- multipart/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/README.md b/multipart/README.md index 17ef46920..0ee8caa4f 100644 --- a/multipart/README.md +++ b/multipart/README.md @@ -34,7 +34,7 @@ via the `tiny_http` feature. Provides server-side integration with `tiny_http::Request` via `multipart::server::Multipart`. -### [Nickel](http://nickel.rs/) ^(returning to `multipart` in 0.14!) +### [Nickel](http://nickel.rs/) returning to `multipart` in 0.14! Provides server-side integration with `&mut nickel::Request` via `multipart::server::Multipart`. From 74e5f2acd7a4099319cae2919ed2020dd2686146 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sun, 4 Feb 2018 18:09:20 -0800 Subject: [PATCH 373/453] two lightning bolt emoji are enough --- multipart/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/README.md b/multipart/README.md index 0ee8caa4f..5aca1b915 100644 --- a/multipart/README.md +++ b/multipart/README.md @@ -39,7 +39,7 @@ Provides server-side integration with `tiny_http::Request` via `multipart::serve Provides server-side integration with `&mut nickel::Request` via `multipart::server::Multipart`. -## âš¡âš¡ Powered By âš¡âš¡ +## âš¡ Powered By âš¡ * [buf_redux ![](https://img.shields.io/crates/v/buf_redux.svg)](https://crates.io/crates/buf_redux) From ae623da32a69a872e254183b5bf511777672ebd9 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sun, 4 Feb 2018 18:40:25 -0800 Subject: [PATCH 374/453] all examples compile, tests failing --- multipart/.travis.yml | 1 + multipart/Cargo.toml | 36 +++++++++++++++--------- multipart/examples/hyper_server.rs | 5 ++-- multipart/examples/iron.rs | 45 ++++++++++-------------------- multipart/examples/tiny_http.rs | 29 +++++++++++++------ multipart/src/server/boundary.rs | 2 +- multipart/src/server/mod.rs | 11 +++++--- 7 files changed, 69 insertions(+), 60 deletions(-) diff --git a/multipart/.travis.yml b/multipart/.travis.yml index 8ee210c95..9573cffb0 100644 --- a/multipart/.travis.yml +++ b/multipart/.travis.yml @@ -20,6 +20,7 @@ env: - ARGS+=--no-default-features # ensure nickel works without hyper - ARGS+='--no-default-features --features "nickel"' + - ARGS+='--features "use-arc-str"' script: - if [ ${TRAVIS_RUST_VERSION} = "nightly" ]; then cargo build --verbose $ARGS --features "nightly"; diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 1734a95c7..a27b4585e 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -48,23 +48,33 @@ bench = [] # Use this to enable SSE4.2 instructions in boundary finding # TODO: Benchmark this sse4 = ["nightly", "twoway/pcmp"] -# switch uses of `Arc` (`From` impl only stabilized in 1.21) for `Arc` +# switch uses of `Arc` for `Arc` (`From` impl only stabilized in 1.21) use_arc_str = [] -[examples.hyper_client] -required_features = ["hyper"] +[[example]] +name = "hyper_client" +required-features = ["mock", "hyper"] -[examples.hyper_server] -required_features = ["hyper"] +[[example]] +name = "hyper_reqbuilder" +required-features = ["mock", "hyper"] -[examples.iron] -required_features = ["iron"] +[[example]] +name = "hyper_server" +required-features = ["mock", "hyper"] -[examples.iron_intercept] -required_features = ["iron"] +[[example]] +name = "iron" +required-features = ["mock", "iron"] -[examples.nickel] -required_features = ["nickel"] +[[example]] +name = "iron_intercept" +required-features = ["mock", "iron"] -[examples.tiny_http] -required_features = ["tiny_http"] +[[example]] +name = "nickel" +required-features = ["mock", "nickel"] + +[[example]] +name = "tiny_http" +required-features = ["mock", "tiny_http"] diff --git a/multipart/examples/hyper_server.rs b/multipart/examples/hyper_server.rs index 404a25593..dc27e382b 100644 --- a/multipart/examples/hyper_server.rs +++ b/multipart/examples/hyper_server.rs @@ -37,8 +37,9 @@ impl MultipartHandler for EchoMultipart { fn process_entries(res: HyperResponse, entries: Entries) -> io::Result<()> { let mut res = res.start()?; - let ref stdout = io::stdout(); - entries.write_debug(StdoutTee::new(&mut res, stdout)) + let stdout = io::stdout(); + let out = StdoutTee::new(&mut res, &stdout); + entries.write_debug(out) } fn main() { diff --git a/multipart/examples/iron.rs b/multipart/examples/iron.rs index 9230b19e3..ee9a8049b 100644 --- a/multipart/examples/iron.rs +++ b/multipart/examples/iron.rs @@ -4,7 +4,8 @@ extern crate iron; extern crate env_logger; use std::fs::File; -use std::io::Read; +use std::io::{self, Read, Write}; +use multipart::mock::StdoutTee; use multipart::server::{Multipart, Entries, SaveResult, SavedField}; use iron::prelude::*; use iron::status; @@ -47,38 +48,20 @@ fn process_request(request: &mut Request) -> IronResult { /// Processes saved entries from multipart request. /// Returns an OK response or an error. fn process_entries(entries: Entries) -> IronResult { - for (name, field) in entries.fields { - println!("Field {:?}: {:?}", name, field); - } - - for (name, files) in entries.files { - println!("Field {:?} has {} files:", name, files.len()); - - for file in files { - print_file(&file)?; - } - } - - Ok(Response::with((status::Ok, "Multipart data is processed"))) -} - -fn print_file(saved_file: &SavedField) -> IronResult<()> { - let mut file = match File::open(&saved_file.path) { - Ok(file) => file, - Err(error) => { - return Err(IronError::new(error, - (status::InternalServerError, - "Server couldn't open saved file"))) - } - }; + let mut data = Vec::new(); - let mut contents = String::new(); - if let Err(error) = file.read_to_string(&mut contents) { - return Err(IronError::new(error, (status::BadRequest, "The file was not a text"))); + { + let stdout = io::stdout(); + let tee = StdoutTee::new(&mut data, &stdout); + entries.write_debug(tee).map_err(|e| { + IronError::new( + e, + (status::InternalServerError, "Error printing request fields") + ) + })?; } - println!("File {:?} ({:?}):", saved_file.filename, saved_file.content_type); - println!("{}", contents); + let _ = writeln!(data, "Entries processed"); - Ok(()) + Ok(Response::with((status::Ok, data))) } diff --git a/multipart/examples/tiny_http.rs b/multipart/examples/tiny_http.rs index e9b0c5a3c..2f61b149a 100644 --- a/multipart/examples/tiny_http.rs +++ b/multipart/examples/tiny_http.rs @@ -2,8 +2,9 @@ extern crate tiny_http; extern crate multipart; use std::fs::File; -use std::io::{self, Read}; +use std::io::{self, Cursor, Read, Write}; use multipart::server::{Multipart, Entries, SaveResult, SavedField}; +use multipart::mock::StdoutTee; use tiny_http::{Response, StatusCode, Request}; fn main() { // Starting a server on `localhost:80` @@ -27,8 +28,10 @@ fn main() { } } +type RespBody = Cursor>; + /// Processes a request and returns response or an occured error. -fn process_request<'a, 'b>(request: &'a mut Request) -> io::Result> { +fn process_request(request: &mut Request) -> io::Result> { // Getting a multipart reader wrapper match Multipart::from_request(request) { Ok(mut multipart) => { @@ -51,18 +54,26 @@ fn process_request<'a, 'b>(request: &'a mut Request) -> io::Result(entries: Entries) -> io::Result> { +fn process_entries<'a>(entries: Entries) -> io::Result> { + let mut data = Vec::new(); + + { + let stdout = io::stdout(); + let tee = StdoutTee::new(&mut data, &stdout); + entries.write_debug(tee)?; + } + writeln!(data, "Entries processed")?; - Ok(build_response(200, "Multipart data is received!")) + Ok(build_response(200, data)) } -/// A utility function to build responses using only two arguments -fn build_response(status_code: u16, response: &str) -> Response<&[u8]> { - let bytes = response.as_bytes(); +fn build_response>>(status_code: u16, data: D) -> Response { + let data = data.into(); + let data_len = data.len(); Response::new(StatusCode(status_code), vec![], - bytes, - Some(bytes.len()), + Cursor::new(data), + Some(data_len), None) } diff --git a/multipart/src/server/boundary.rs b/multipart/src/server/boundary.rs index cd37b6ba7..748d81bc9 100644 --- a/multipart/src/server/boundary.rs +++ b/multipart/src/server/boundary.rs @@ -127,7 +127,7 @@ impl BoundaryReader where R: Read { let consume_amt = { let min_len = self.boundary.len() + 4; - let buf = fill_buf_min(&mut self.source, min_len)?; + let buf = self.source.fill_buf()?; if buf.len() < min_len { return Err(io::Error::new(io::ErrorKind::UnexpectedEof, diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index a97c64aef..cb2ac1d24 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -35,14 +35,17 @@ pub use self::save::{Entries, SaveResult, SavedField}; use self::save::EntriesSaveResult; -/// Replacement typedef for `Arc` for older Rust releases. +/// Default typedef for shared strings. +/// +/// Enable the `use_arc_str` feature to use `Arc` instead, which saves an indirection but +/// cannot be constructed in Rust versions older than 1.21 (the `From` impl was stabilized +/// in that release). #[cfg(not(feature = "use_arc_str"))] pub type ArcStr = Arc; -/// Typedef for `Arc`. +/// Optimized typedef for shared strings, replacing `Arc`. /// -/// Construction of `Arc` was only stabilized in Rust 1.21, so to continue to support -/// older versions, an alternate typedef of `Arc` is available under the `no_arc_str` feature. +/// Enabled with the `use_arc_str` feature. #[cfg(feature = "use_arc_str")] pub type ArcStr = Arc; From 9db779351a5d171b3974cdc8d9be2d8848722c79 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sun, 4 Feb 2018 18:42:18 -0800 Subject: [PATCH 375/453] powered-by: convert list to h5s --- multipart/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/multipart/README.md b/multipart/README.md index 5aca1b915..8a520956c 100644 --- a/multipart/README.md +++ b/multipart/README.md @@ -41,18 +41,18 @@ Provides server-side integration with `&mut nickel::Request` via `multipart::ser ## âš¡ Powered By âš¡ -* [buf_redux ![](https://img.shields.io/crates/v/buf_redux.svg)](https://crates.io/crates/buf_redux) +##### [buf_redux ![](https://img.shields.io/crates/v/buf_redux.svg)](https://crates.io/crates/buf_redux) Customizable drop-in `std::io::BufReader` replacement, created to be used in this crate. Needed because it can read more bytes into the buffer without the buffer being empty, necessary when a boundary falls across two reads. (It was easier to author a new crate than try to get this added to `std::io::BufReader`.) -* [httparse ![](https://img.shields.io/crates/v/httparse.svg)](https://crates.io/crates/httparse) +##### [httparse ![](https://img.shields.io/crates/v/httparse.svg)](https://crates.io/crates/httparse) Fast, zero-copy HTTP header parsing, used to read field headers in `multipart/form-data` request bodies. -* [twoway ![](https://img.shields.io/crates/v/twoway.svg)](https://crates.io/crates/twoway) +##### [twoway ![](https://img.shields.io/crates/v/twoway.svg)](https://crates.io/crates/twoway) Fast string and byte-string search. Used to find boundaries in the request body. SSE 4.2 acceleration available under the `sse42` or `twoway/pcmp` features. From 55b160ed3cc054f17425efa83b4632744d589c90 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sun, 4 Feb 2018 18:42:37 -0800 Subject: [PATCH 376/453] powered-by: convert list to h4s --- multipart/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/multipart/README.md b/multipart/README.md index 8a520956c..67476602d 100644 --- a/multipart/README.md +++ b/multipart/README.md @@ -41,7 +41,7 @@ Provides server-side integration with `&mut nickel::Request` via `multipart::ser ## âš¡ Powered By âš¡ -##### [buf_redux ![](https://img.shields.io/crates/v/buf_redux.svg)](https://crates.io/crates/buf_redux) +#### [buf_redux ![](https://img.shields.io/crates/v/buf_redux.svg)](https://crates.io/crates/buf_redux) Customizable drop-in `std::io::BufReader` replacement, created to be used in this crate. Needed because it can read more bytes into the buffer without the buffer being empty, necessary @@ -52,7 +52,7 @@ to `std::io::BufReader`.) Fast, zero-copy HTTP header parsing, used to read field headers in `multipart/form-data` request bodies. -##### [twoway ![](https://img.shields.io/crates/v/twoway.svg)](https://crates.io/crates/twoway) +#### [twoway ![](https://img.shields.io/crates/v/twoway.svg)](https://crates.io/crates/twoway) Fast string and byte-string search. Used to find boundaries in the request body. SSE 4.2 acceleration available under the `sse42` or `twoway/pcmp` features. From b2dbe9adb29a11da9ca8ba02a8b7e8533751f350 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sun, 4 Feb 2018 18:43:21 -0800 Subject: [PATCH 377/453] Update README.md --- multipart/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/multipart/README.md b/multipart/README.md index 67476602d..0be67da5d 100644 --- a/multipart/README.md +++ b/multipart/README.md @@ -41,18 +41,18 @@ Provides server-side integration with `&mut nickel::Request` via `multipart::ser ## âš¡ Powered By âš¡ -#### [buf_redux ![](https://img.shields.io/crates/v/buf_redux.svg)](https://crates.io/crates/buf_redux) +### [buf_redux ![](https://img.shields.io/crates/v/buf_redux.svg)](https://crates.io/crates/buf_redux) Customizable drop-in `std::io::BufReader` replacement, created to be used in this crate. Needed because it can read more bytes into the buffer without the buffer being empty, necessary when a boundary falls across two reads. (It was easier to author a new crate than try to get this added to `std::io::BufReader`.) -##### [httparse ![](https://img.shields.io/crates/v/httparse.svg)](https://crates.io/crates/httparse) +### [httparse ![](https://img.shields.io/crates/v/httparse.svg)](https://crates.io/crates/httparse) Fast, zero-copy HTTP header parsing, used to read field headers in `multipart/form-data` request bodies. -#### [twoway ![](https://img.shields.io/crates/v/twoway.svg)](https://crates.io/crates/twoway) +### [twoway ![](https://img.shields.io/crates/v/twoway.svg)](https://crates.io/crates/twoway) Fast string and byte-string search. Used to find boundaries in the request body. SSE 4.2 acceleration available under the `sse42` or `twoway/pcmp` features. From ea9ba5d68a281092e886a321f4f8a5a958ecf87f Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sun, 4 Feb 2018 18:45:22 -0800 Subject: [PATCH 378/453] Add crates.io badges to integrations, alphabetize --- multipart/README.md | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/multipart/README.md b/multipart/README.md index 0be67da5d..8cb5d98b9 100644 --- a/multipart/README.md +++ b/multipart/README.md @@ -13,7 +13,7 @@ Minimum supported Rust version: 1.17.0 Example files demonstrating how to use `multipart` with these crates are available under [`examples/`](examples). -### [Hyper](http://hyper.rs) +### [Hyper ![](https://img.shields.io/crates/v/hyper.svg)](https://crates.io/crates/hyper) via the `hyper` feature (enabled by default). **Note: Hyper 0.9, 0.10 (synchronous API) only**; support for asynchronous APIs will be provided by [multipart-async]. @@ -23,21 +23,20 @@ as integration with the new `hyper::Client` API via `multipart::client::lazy::Mu Server integration for `hyper::server::Request` via `multipart::server::Multipart`. -### [Iron](http://ironframework.io) +### [Iron ![](https://img.shields.io/crates/v/iron.svg)](https://crates.io/crates/iron) via the `iron` feature. Provides regular server-side integration with `iron::Request` via `multipart::server::Multipart`, as well as a convenient `BeforeMiddleware` implementation in `multipart::server::iron::Intercept`. -### [tiny\_http](https://crates.io/crates/tiny_http/) -via the `tiny_http` feature. - -Provides server-side integration with `tiny_http::Request` via `multipart::server::Multipart`. - -### [Nickel](http://nickel.rs/) returning to `multipart` in 0.14! +### [Nickel ![](https://img.shields.io/crates/v/nickel.svg)](https://crates.io/crates/nickel) returning to `multipart` in 0.14! Provides server-side integration with `&mut nickel::Request` via `multipart::server::Multipart`. +### [tiny_http ![](https://img.shields.io/crates/v/tiny_http.svg)](https://crates.io/crates/tiny_http) +via the `tiny_http` feature. + +Provides server-side integration with `tiny_http::Request` via `multipart::server::Multipart`. ## âš¡ Powered By âš¡ From a9c370f25f0a107952e1d439e16325fc903194f0 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sun, 4 Feb 2018 18:48:53 -0800 Subject: [PATCH 379/453] fix notes about Nickel integration --- multipart/README.md | 1 + multipart/src/lib.rs | 12 +++++++----- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/multipart/README.md b/multipart/README.md index 8cb5d98b9..e58e23d91 100644 --- a/multipart/README.md +++ b/multipart/README.md @@ -30,6 +30,7 @@ Provides regular server-side integration with `iron::Request` via `multipart::se as well as a convenient `BeforeMiddleware` implementation in `multipart::server::iron::Intercept`. ### [Nickel ![](https://img.shields.io/crates/v/nickel.svg)](https://crates.io/crates/nickel) returning to `multipart` in 0.14! +via the `nickel` feature. Provides server-side integration with `&mut nickel::Request` via `multipart::server::Multipart`. diff --git a/multipart/src/lib.rs b/multipart/src/lib.rs index 97e01106e..6ccdaff4a 100644 --- a/multipart/src/lib.rs +++ b/multipart/src/lib.rs @@ -16,16 +16,18 @@ //! * `mock`: Provides mock implementations of core `client` and `server` traits for debugging //! or non-standard use. //! -//! * `hyper`: Integration with the [Hyper](https://github.com/hyperium/hyper) HTTP library +//! * `hyper`: Integration with the [Hyper](https://crates.io/crates/hyper) HTTP library //! for client and/or server depending on which other feature flags are set. //! -//! * `iron`: Integration with the [Iron](http://ironframework.io) web application +//! * `iron`: Integration with the [Iron](http://crates.io/crates/iron) web application //! framework. See the [`server::iron`](server/iron/index.html) module for more information. //! -//! * `tiny_http`: Integration with the [`tiny_http`](https://github.com/frewsxcv/tiny-http) -//! crate. See the [`server::tiny_http`](server/tiny_http/index.html) module for more information. +//! * `nickel` (returning in 0.14!): Integration with the [Nickel](https://crates.io/crates/nickel) +//! web application framework. See the [`server::nickel`](server/nickel/index.html) module for more +//! information. //! -//! `nickel` (returning in 0.14!): Integration with the See the [`server::nickel`](server/nickel/index.html) module for more information. +//! * `tiny_http`: Integration with the [`tiny_http`](https://crates.io/crates/tiny_http) +//! crate. See the [`server::tiny_http`](server/tiny_http/index.html) module for more information. //! //! ### Note: Work in Progress //! I have left a number of Request-for-Comments (RFC) questions on various APIs and other places From bfb111187af487969dd35ef9afdd62493c153cd0 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sun, 4 Feb 2018 18:50:15 -0800 Subject: [PATCH 380/453] add `readme` to Cargo.toml --- multipart/Cargo.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index a27b4585e..ca5e10c57 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -15,6 +15,8 @@ documentation = "http://docs.rs/multipart/" license = "MIT OR Apache-2.0" +readme = "README.md" + [dependencies] log = "0.3" mime = "0.3" From 0533108181ca5cc90a4d392c34d0a53e7804e387 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sun, 4 Feb 2018 19:32:22 -0800 Subject: [PATCH 381/453] fix `local_test` --- multipart/src/local_test.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/multipart/src/local_test.rs b/multipart/src/local_test.rs index 04150a989..da7985205 100644 --- a/multipart/src/local_test.rs +++ b/multipart/src/local_test.rs @@ -72,7 +72,8 @@ impl TestFields { } fn check_field(&mut self, mut field: MultipartField) -> M { - if field.is_text() { + // text/plain fields would be considered a file by `TestFields` + if field.headers.content_type.is_none() { let mut text_entries = expect_fmt!(get_field(&field.headers, &mut self.texts), "Got text field that wasn't in original dataset: {:?}", field.headers); From 758b370e8656a4ffab8d2817b3de2fd19822a459 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sun, 4 Feb 2018 19:45:50 -0800 Subject: [PATCH 382/453] remove `use_arc_str` from the default features --- multipart/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index ca5e10c57..2a46e497c 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -42,7 +42,7 @@ env_logger = "0.3" [features] client = [] -default = ["client", "hyper", "iron", "mock", "nickel", "server", "tiny_http", "use_arc_str"] +default = ["client", "hyper", "iron", "mock", "nickel", "server", "tiny_http"] server = ["buf_redux", "httparse", "safemem", "twoway"] mock = [] nightly = [] From e92112edb6221d7f3d518ee92854ab12314597e9 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sun, 4 Feb 2018 19:46:41 -0800 Subject: [PATCH 383/453] set `multipart=trace` for logs --- multipart/.travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/.travis.yml b/multipart/.travis.yml index 9573cffb0..324422e71 100644 --- a/multipart/.travis.yml +++ b/multipart/.travis.yml @@ -15,7 +15,7 @@ os: # - osx env: global: - - RUST_LOG=multipart=info RUST_BACKTRACE=1 ARGS= + - RUST_LOG=multipart=trace RUST_BACKTRACE=1 ARGS= matrix: - ARGS+=--no-default-features # ensure nickel works without hyper From ad7653cfee7fe2402af9aba60400d8f6a47c4378 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sun, 4 Feb 2018 20:15:43 -0800 Subject: [PATCH 384/453] add `--test-threads=1` so output is serialized --- multipart/.travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/multipart/.travis.yml b/multipart/.travis.yml index 324422e71..5913dd928 100644 --- a/multipart/.travis.yml +++ b/multipart/.travis.yml @@ -24,9 +24,9 @@ env: script: - if [ ${TRAVIS_RUST_VERSION} = "nightly" ]; then cargo build --verbose $ARGS --features "nightly"; - cargo test --verbose $ARGS --features "nightly"; + cargo test --verbose $ARGS --features "nightly" -- --test-threads=1; cargo doc --verbose $ARGS --features "nightly"; fi - cargo build --verbose $ARGS; - - cargo test --verbose $ARGS; + - cargo test --verbose $ARGS -- --test-threads=1; - cargo doc --verbose $ARGS; From f8439aede07816ca0164f3cb4267c00da4da0703 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sun, 4 Feb 2018 21:15:56 -0800 Subject: [PATCH 385/453] fix behavior with buf_redux strategies --- multipart/src/server/boundary.rs | 13 +++++- multipart/src/server/field.rs | 70 +++++++++++++++++--------------- multipart/src/server/mod.rs | 6 ++- 3 files changed, 54 insertions(+), 35 deletions(-) diff --git a/multipart/src/server/boundary.rs b/multipart/src/server/boundary.rs index 748d81bc9..3b5e01866 100644 --- a/multipart/src/server/boundary.rs +++ b/multipart/src/server/boundary.rs @@ -23,6 +23,8 @@ use std::io::prelude::*; use self::State::*; +pub const MIN_BUF_SIZE: usize = 1024; + #[derive(Debug, PartialEq, Eq)] enum State { Searching, @@ -48,8 +50,8 @@ impl BoundaryReader where R: Read { BoundaryReader { source: BufReader::with_strategies( reader, - LessThan(boundary.len() * 2), - AtEndLessThan(boundary.len() * 2), + LessThan(MIN_BUF_SIZE), + AtEndLessThan(MIN_BUF_SIZE), ), boundary, search_idx: 0, @@ -108,6 +110,13 @@ impl BoundaryReader where R: Read { Ok(ret_buf) } + pub fn set_min_buf_size(&mut self, min_buf_size: usize) { + let min_buf_size = cmp::min(self.boundary.len() * 2, min_buf_size); + + self.source.read_strategy_mut().0 = min_buf_size; + self.source.move_strategy_mut().0 = min_buf_size; + } + #[doc(hidden)] pub fn consume_boundary(&mut self) -> io::Result { if self.state == AtEnd { diff --git a/multipart/src/server/field.rs b/multipart/src/server/field.rs index 56b9d718a..2ea59df34 100644 --- a/multipart/src/server/field.rs +++ b/multipart/src/server/field.rs @@ -23,7 +23,6 @@ use super::save::{SaveBuilder, SavedField}; use super::ArcStr; - const EMPTY_STR_HEADER: StrHeader<'static> = StrHeader { name: "", val: "", @@ -42,37 +41,25 @@ fn with_headers(r: &mut R, closure: F) -> Result Ret { const HEADER_LEN: usize = 4; - // These are only written once so they don't need to be `mut` or initialized. - let consume; - let ret; - - let mut attempts = 0; - - loop { - let mut raw_headers = [EMPTY_HEADER; HEADER_LEN]; - + let (consume, ret) = { let buf = r.fill_buf()?; - if attempts == MAX_ATTEMPTS { - return Err(ParseHeaderError::Other("Could not read field headers".to_string())); - } + let mut raw_headers = [EMPTY_HEADER; HEADER_LEN]; match httparse::parse_headers(buf, &mut raw_headers) { - Ok(Status::Complete((consume_, raw_headers))) => { - consume = consume_; + Err(err) => return Err(ParseHeaderError::from(err)), + Ok(Status::Partial) => return Err( + // FIXME: make this a variant + ParseHeaderError::Other("Field headers too large".to_string()) + ), + Ok(Status::Complete((consume, raw_headers))) => { let mut headers = [EMPTY_STR_HEADER; HEADER_LEN]; let headers = copy_headers(raw_headers, &mut headers)?; debug!("Parsed headers: {:?}", headers); - ret = closure(headers); - break; - }, - Ok(Status::Partial) => { - attempts += 1; - continue; + (consume, closure(headers)) }, - Err(err) => return Err(ParseHeaderError::from(err)), - }; - } + } + }; r.consume(consume); Ok(ret) @@ -271,6 +258,16 @@ impl MultipartData where M: ReadEntry { self.inner.expect(DATA_INNER_ERR) } + /// Set the minimum buffer size that `BufRead::fill_buf(self)` will return + /// until the end of the stream is reached. Set this as small as you can tolerate + /// to minimize `read()` calls (`read()` won't be called again until the buffer + /// is smaller than this). + /// + /// This value is reset between fields. + pub fn set_min_buf_size(&mut self, min_buf_size: usize) { + self.inner_mut().set_min_buf_size(min_buf_size) + } + fn inner_mut(&mut self) -> &mut M { self.inner.as_mut().expect(DATA_INNER_ERR) } @@ -286,17 +283,18 @@ impl MultipartData where M: ReadEntry { impl Read for MultipartData { fn read(&mut self, buf: &mut [u8]) -> io::Result{ - self.inner_mut().source().read(buf) + self.inner_mut().source_mut().read(buf) } } +/// Unlike `std::io::BufReader`, impl BufRead for MultipartData { fn fill_buf(&mut self) -> io::Result<&[u8]> { - self.inner_mut().source().fill_buf() + self.inner_mut().source_mut().fill_buf() } fn consume(&mut self, amt: usize) { - self.inner_mut().source().consume(amt) + self.inner_mut().source_mut().consume(amt) } } @@ -334,6 +332,8 @@ fn find_header<'a, 'b>(headers: &'a [StrHeader<'b>], name: &str) -> Option<&'a S pub trait ReadEntry: PrivReadEntry + Sized { /// Attempt to read the next entry in the multipart stream. fn read_entry(mut self) -> ReadEntryResult { + self.set_min_buf_size(super::boundary::MIN_BUF_SIZE); + debug!("ReadEntry::read_entry()"); if try_read_entry!(self; self.consume_boundary()) { @@ -374,21 +374,23 @@ impl ReadEntry for T where T: PrivReadEntry {} pub trait PrivReadEntry { type Source: BufRead; - fn source(&mut self) -> &mut Self::Source; + fn source_mut(&mut self) -> &mut Self::Source; + + fn set_min_buf_size(&mut self, min_buf_size: usize); /// Consume the next boundary. /// Returns `true` if the last boundary was read, `false` otherwise. fn consume_boundary(&mut self) -> io::Result; fn read_headers(&mut self) -> Result { - FieldHeaders::read_from(&mut self.source()) + FieldHeaders::read_from(self.source_mut()) .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e)) } fn read_to_string(&mut self) -> io::Result { let mut buf = String::new(); - match self.source().read_to_string(&mut buf) { + match self.source_mut().read_to_string(&mut buf) { Ok(_) => Ok(buf), Err(err) => Err(err), } @@ -398,8 +400,12 @@ pub trait PrivReadEntry { impl<'a, M: ReadEntry> PrivReadEntry for &'a mut M { type Source = M::Source; - fn source(&mut self) -> &mut M::Source { - (**self).source() + fn source_mut(&mut self) -> &mut M::Source { + (**self).source_mut() + } + + fn set_min_buf_size(&mut self, min_buf_size: usize) { + (**self).set_min_buf_size(min_buf_size) } fn consume_boundary(&mut self) -> io::Result { diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index cb2ac1d24..284df9b4c 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -184,10 +184,14 @@ impl Borrow for Multipart { impl PrivReadEntry for Multipart { type Source = BoundaryReader; - fn source(&mut self) -> &mut BoundaryReader { + fn source_mut(&mut self) -> &mut BoundaryReader { &mut self.reader } + fn set_min_buf_size(&mut self, min_buf_size: usize) { + self.reader.set_min_buf_size(min_buf_size) + } + /// Consume the next boundary. /// Returns `true` if the last boundary was read, `false` otherwise. fn consume_boundary(&mut self) -> io::Result { From 0396d9fee48e225a98a12233da0efbc20da2a235 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Mon, 5 Feb 2018 17:18:25 -0800 Subject: [PATCH 386/453] field: improve parse error handling --- multipart/Cargo.toml | 3 +- multipart/src/lib.rs | 4 + multipart/src/server/field.rs | 178 +++++++++++++++++----------------- 3 files changed, 95 insertions(+), 90 deletions(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 2a46e497c..f9ef14b1d 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -30,6 +30,7 @@ clippy = { version = ">=0.0, <0.1", optional = true} buf_redux = { version = "0.6", optional = true } httparse = { version = "1.2", optional = true } twoway = { version = "0.1", optional = true } +quick-error = { version = "1.2", optional = true } # Optional Integrations hyper = { version = ">=0.9, <0.11", optional = true, default-features = false } @@ -43,7 +44,7 @@ env_logger = "0.3" [features] client = [] default = ["client", "hyper", "iron", "mock", "nickel", "server", "tiny_http"] -server = ["buf_redux", "httparse", "safemem", "twoway"] +server = ["buf_redux", "httparse", "quick-error", "safemem", "twoway"] mock = [] nightly = [] bench = [] diff --git a/multipart/src/lib.rs b/multipart/src/lib.rs index 6ccdaff4a..6c6972b7b 100644 --- a/multipart/src/lib.rs +++ b/multipart/src/lib.rs @@ -55,6 +55,10 @@ extern crate mime_guess; extern crate rand; extern crate tempdir; +#[cfg(feature = "quick-error")] +#[macro_use] +extern crate quick_error; + #[cfg(feature = "server")] extern crate safemem; diff --git a/multipart/src/server/field.rs b/multipart/src/server/field.rs index 2ea59df34..cca789f6d 100644 --- a/multipart/src/server/field.rs +++ b/multipart/src/server/field.rs @@ -6,8 +6,11 @@ // copied, modified, or distributed except according to those terms. //! `multipart` field header parsing. -use mime::{self, Mime}; +use mime::{self, Mime, FromStrError as MimeParseErr}; +use quick_error::ResultExt; + +use std::error::Error; use std::io::{self, Read, BufRead, Write}; use std::ops::Deref; use std::path::{Path, PathBuf}; @@ -15,7 +18,7 @@ use std::{str, fmt, error}; use std::ascii::AsciiExt; -use super::httparse::{self, EMPTY_HEADER, Header, Status}; +use super::httparse::{self, EMPTY_HEADER, Header, Status, Error as HttparseError}; use self::ReadEntryResult::*; @@ -28,6 +31,14 @@ const EMPTY_STR_HEADER: StrHeader<'static> = StrHeader { val: "", }; +macro_rules! invalid_cont_disp { + ($reason: expr, $cause: expr) => { + return Err( + ParseHeaderError::InvalidContDisp($reason, $cause.to_string()) + ); + } +} + /// Not exposed #[derive(Copy, Clone, Debug)] pub struct StrHeader<'a> { @@ -35,6 +46,18 @@ pub struct StrHeader<'a> { val: &'a str, } +struct DisplayHeaders<'s, 'a: 's>(&'s [StrHeader<'a>]); + +impl <'s, 'a: 's> fmt::Display for DisplayHeaders<'s, 'a> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + for hdr in self.0 { + writeln!(f, "{}: {}", hdr.name, hdr.val)?; + } + + Ok(()) + } +} + const MAX_ATTEMPTS: usize = 30; fn with_headers(r: &mut R, closure: F) -> Result @@ -46,13 +69,9 @@ where R: BufRead, F: FnOnce(&[StrHeader]) -> Ret { let mut raw_headers = [EMPTY_HEADER; HEADER_LEN]; - match httparse::parse_headers(buf, &mut raw_headers) { - Err(err) => return Err(ParseHeaderError::from(err)), - Ok(Status::Partial) => return Err( - // FIXME: make this a variant - ParseHeaderError::Other("Field headers too large".to_string()) - ), - Ok(Status::Complete((consume, raw_headers))) => { + match httparse::parse_headers(buf, &mut raw_headers)? { + Status::Partial => return Err(ParseHeaderError::TooLarge), + Status::Complete((consume, raw_headers)) => { let mut headers = [EMPTY_STR_HEADER; HEADER_LEN]; let headers = copy_headers(raw_headers, &mut headers)?; debug!("Parsed headers: {:?}", headers); @@ -104,7 +123,8 @@ impl FieldHeaders { } fn parse(headers: &[StrHeader]) -> Result { - let cont_disp = ContentDisp::parse(headers)?.ok_or(ParseHeaderError::MissingContentDisposition)?; + let cont_disp = ContentDisp::parse_required(headers)?; + Ok(FieldHeaders { name: cont_disp.field_name.into(), filename: cont_disp.filename, @@ -122,58 +142,54 @@ struct ContentDisp { } impl ContentDisp { - fn parse(headers: &[StrHeader]) -> Result, ParseHeaderError> { - const CONT_DISP: &'static str = "Content-Disposition"; - let header = if let Some(header) = find_header(headers, CONT_DISP) { + fn parse_required(headers: &[StrHeader]) -> Result { + let header = if let Some(header) = find_header(headers, "Content-Disposition") { header } else { - return Ok(None); + return Err(ParseHeaderError::MissingContentDisposition( + DisplayHeaders(headers).to_string() + )); }; - const NAME: &'static str = "name="; - const FILENAME: &'static str = "filename="; - + // Content-Disposition: ? let after_disp_type = match split_once(header.val, ';') { Some((disp_type, after_disp_type)) => { + // assert Content-Disposition: form-data + // but needs to be parsed out to trim the spaces (allowed by spec IIRC) if disp_type.trim() != "form-data" { - let err = format!("Unexpected Content-Disposition value: {:?}", disp_type); - return Err(ParseHeaderError::Invalid(err)); + invalid_cont_disp!("unexpected Content-Disposition value", disp_type); } after_disp_type }, - None => { - let err = format!("Expected additional data after Content-Disposition type, got {:?}", header.val); - return Err(ParseHeaderError::Invalid(err)); - } + None => invalid_cont_disp!("expected additional data after Content-Disposition type", + header.val), }; - let (field_name, filename) = match get_str_after(NAME, ';', after_disp_type) { - None => { - let err = format!("Expected field name and maybe filename, got {:?}", after_disp_type); - return Err(ParseHeaderError::Invalid(err)); - }, + // Content-Disposition: form-data; name=? + let (field_name, filename) = match get_str_after("name=", ';', after_disp_type) { + None => invalid_cont_disp!("expected field name and maybe filename, got", + after_disp_type), + // Content-Disposition: form-data; name={field_name}; filename=? Some((field_name, after_field_name)) => { let field_name = trim_quotes(field_name); - let filename = get_str_after(FILENAME, ';', after_field_name).map(|(filename, _)| trim_quotes(filename).to_owned()); + let filename = get_str_after("filename=", ';', after_field_name) + .map(|(filename, _)| trim_quotes(filename).to_owned()); (field_name, filename) }, }; - Ok(Some(ContentDisp { field_name: field_name.to_owned(), filename: filename })) + Ok(ContentDisp { field_name: field_name.to_owned(), filename }) } } fn parse_content_type(headers: &[StrHeader]) -> Result, ParseHeaderError> { - const CONTENT_TYPE: &'static str = "Content-Type"; - let header = if let Some(header) = find_header(headers, CONTENT_TYPE) { - header + if let Some(header) = find_header(headers, "Content-Type") { + // Boundary parameter will be parsed into the `Mime` + debug!("Found Content-Type: {:?}", header.val); + Ok(Some(header.val.parse::().context(header.val)?)) } else { - return Ok(None) - }; - - // Boundary parameter will be parsed into the `Mime` - debug!("Found Content-Type: {:?}", header.val); - Ok(Some(read_content_type(header.val.trim()))) + Ok(None) + } } /// A field in a multipart request with its associated headers and data. @@ -298,10 +314,6 @@ impl BufRead for MultipartData { } } -fn read_content_type(cont_type: &str) -> Mime { - cont_type.parse().ok().unwrap_or_else(::mime_guess::octet_stream) -} - fn split_once(s: &str, delim: char) -> Option<(&str, &str)> { s.find(delim).map(|idx| s.split_at(idx)) } @@ -484,56 +496,44 @@ impl ReadEntryResult { } } +const GENERIC_PARSE_ERR: &'static str = "an error occurred while parsing field headers"; -#[derive(Debug)] -enum ParseHeaderError { - /// The `Content-Disposition` header was not found - MissingContentDisposition, - /// The header was found but could not be parsed - Invalid(String), - /// IO error - Io(io::Error), - Other(String), -} - -impl fmt::Display for ParseHeaderError { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - match *self { - ParseHeaderError::MissingContentDisposition => write!(f, "\"Content-Disposition\" header not found (ParseHeaderError::MissingContentDisposition)"), - ParseHeaderError::Invalid(ref msg) => write!(f, "invalid header (ParseHeaderError::Invalid({}))", msg), - ParseHeaderError::Io(_) => write!(f, "could not read header (ParseHeaderError::Io)"), - ParseHeaderError::Other(ref reason) => write!(f, "unknown parsing error (ParseHeaderError::Other(\"{}\"))", reason), +quick_error! { + #[derive(Debug)] + enum ParseHeaderError { + /// The `Content-Disposition` header was not found + MissingContentDisposition(headers: String) { + display(x) -> ("{}:\n{}", x.description(), headers) + description("\"Content-Disposition\" header not found in field headers") } - } -} - -impl error::Error for ParseHeaderError { - fn description(&self) -> &str { - match *self { - ParseHeaderError::MissingContentDisposition => "\"Content-Disposition\" header not found", - ParseHeaderError::Invalid(_) => "the header is not formatted correctly", - ParseHeaderError::Io(_) => "failed to read the header", - ParseHeaderError::Other(_) => "unknown parsing error", + InvalidContDisp(reason: &'static str, cause: String) { + display(x) -> ("{}: {}: {}", x.description(), reason, cause) + description("invalid \"Content-Disposition\" header") } - } - - fn cause(&self) -> Option<&error::Error> { - match *self { - ParseHeaderError::Io(ref e) => Some(e), - _ => None, + /// The header was found but could not be parsed + TokenizeError(err: HttparseError) { + description(GENERIC_PARSE_ERR) + display(x) -> ("{}: {}", x.description(), err) + cause(err) + from() + } + MimeError(err: MimeParseErr, cont_type: String) { + description(err.description()) + display("{}\nwhile parsing Content-Type value: {}", err.description(), cont_type) + cause(err) + context(cont_type: &'a str, err: MimeParseErr) -> + (err, cont_type.to_string()) + } + TooLarge { + description("field headers section ridiculously long or missing trailing CRLF-CRLF") + } + /// IO error + Io(err: io::Error) { + description("an io error occurred while parsing the headers") + display(x) -> ("{}: {}", x.description(), err) + cause(err) + from() } - } -} - -impl From for ParseHeaderError { - fn from(err: io::Error) -> ParseHeaderError { - ParseHeaderError::Io(err) - } -} - -impl From for ParseHeaderError { - fn from(err: httparse::Error) -> ParseHeaderError { - ParseHeaderError::Invalid(format!("{}", err)) } } From 952dd9db0a1de344dff36dbf970b37d4791d8904 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Mon, 5 Feb 2018 20:52:56 -0800 Subject: [PATCH 387/453] fix `BoundaryReader::set_min_buf_size` --- multipart/src/server/boundary.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/multipart/src/server/boundary.rs b/multipart/src/server/boundary.rs index 3b5e01866..0d7cdac2b 100644 --- a/multipart/src/server/boundary.rs +++ b/multipart/src/server/boundary.rs @@ -111,7 +111,8 @@ impl BoundaryReader where R: Read { } pub fn set_min_buf_size(&mut self, min_buf_size: usize) { - let min_buf_size = cmp::min(self.boundary.len() * 2, min_buf_size); + // ensure the minimum buf size is at least enough to find a boundary with some extra + let min_buf_size = cmp::max(self.boundary.len() * 2, min_buf_size); self.source.read_strategy_mut().0 = min_buf_size; self.source.move_strategy_mut().0 = min_buf_size; From 5b4c4df4d9f59d63e40a700cb6b1712b28bde96b Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Thu, 8 Feb 2018 05:13:00 -0800 Subject: [PATCH 388/453] fix spuriously failing test, improve logging in `local_test` by buffering logs until a test panics --- multipart/Cargo.toml | 4 +- multipart/src/bin/test_multipart.rs | 19 ++++---- multipart/src/client/lazy.rs | 3 -- multipart/src/lib.rs | 3 -- multipart/src/local_test.rs | 6 ++- multipart/src/mock.rs | 69 ++++++++++++++++++++++++++++- multipart/src/server/boundary.rs | 32 +++++++++---- multipart/src/server/field.rs | 28 +++++++++--- 8 files changed, 129 insertions(+), 35 deletions(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index f9ef14b1d..191f0f085 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -18,7 +18,7 @@ license = "MIT OR Apache-2.0" readme = "README.md" [dependencies] -log = "0.3" +log = "0.4" mime = "0.3" mime_guess = "2.0.0-alpha.3" rand = "0.3" @@ -39,7 +39,7 @@ tiny_http = { version = "0.5", optional = true } nickel = { version = ">=0.10.1", optional = true } [dev-dependencies] -env_logger = "0.3" +env_logger = "0.4" [features] client = [] diff --git a/multipart/src/bin/test_multipart.rs b/multipart/src/bin/test_multipart.rs index d8dbaa956..0a55a0b89 100644 --- a/multipart/src/bin/test_multipart.rs +++ b/multipart/src/bin/test_multipart.rs @@ -2,8 +2,6 @@ extern crate multipart; extern crate rand; -use log::{LogRecord, LogMetadata, LogLevelFilter}; - use multipart::server::Multipart; use rand::{Rng, ThreadRng}; @@ -12,28 +10,29 @@ use std::fs::File; use std::env; use std::io::{self, Read}; -const LOG_LEVEL: LogLevelFilter = LogLevelFilter::Debug; +const LOG_LEVEL: log::LevelFilter = log::LevelFilter::Debug; struct SimpleLogger; impl log::Log for SimpleLogger { - fn enabled(&self, metadata: &LogMetadata) -> bool { - LOG_LEVEL.to_log_level() + fn enabled(&self, metadata: &log::Metadata) -> bool { + LOG_LEVEL.to_level() .map_or(false, |level| metadata.level() <= level) } - fn log(&self, record: &LogRecord) { + fn log(&self, record: &log::Record) { if self.enabled(record.metadata()) { println!("{} - {}", record.level(), record.args()); } } + + fn flush(&self) {} } +static LOGGER: SimpleLogger = SimpleLogger; + fn main() { - log::set_logger(|max_log_level| { - max_log_level.set(LOG_LEVEL); - Box::new(SimpleLogger) - }).expect("Could not initialize logger"); + log::set_logger(&LOGGER).expect("Could not initialize logger"); let mut args = env::args().skip(1); diff --git a/multipart/src/client/lazy.rs b/multipart/src/client/lazy.rs index 2eef46caf..a33ce46a2 100644 --- a/multipart/src/client/lazy.rs +++ b/multipart/src/client/lazy.rs @@ -1,7 +1,4 @@ //! Multipart requests which write out their data in one fell swoop. - -use log::LogLevel; - use mime::Mime; use std::borrow::Cow; diff --git a/multipart/src/lib.rs b/multipart/src/lib.rs index 6c6972b7b..cde8cdef4 100644 --- a/multipart/src/lib.rs +++ b/multipart/src/lib.rs @@ -45,9 +45,6 @@ #[macro_use] extern crate log; -#[cfg(test)] -extern crate env_logger; - #[cfg_attr(test, macro_use)] extern crate mime; diff --git a/multipart/src/local_test.rs b/multipart/src/local_test.rs index da7985205..0bf3d1430 100644 --- a/multipart/src/local_test.rs +++ b/multipart/src/local_test.rs @@ -205,7 +205,7 @@ impl fmt::Debug for PrintHex { macro_rules! do_test ( ($client_test:ident, $server_test:ident) => ( - let _ = ::env_logger::init(); + let panic_logger = ::mock::log_on_panic(); info!("Client Test: {:?} Server Test: {:?}", stringify!($client_test), stringify!($server_test)); @@ -224,6 +224,8 @@ macro_rules! do_test ( $server_test(buf, &mut test_fields); test_fields.assert_is_empty(); + + panic_logger.clear(); ); ); @@ -252,7 +254,7 @@ mod extended { use std::time::Instant; - const TIME_LIMIT_SECS: u64 = 300; + const TIME_LIMIT_SECS: u64 = 600; #[test] #[ignore] diff --git a/multipart/src/mock.rs b/multipart/src/mock.rs index cee48b369..0581b96c2 100644 --- a/multipart/src/mock.rs +++ b/multipart/src/mock.rs @@ -5,8 +5,12 @@ // http://opensource.org/licenses/MIT>, at your option. This file may not be // copied, modified, or distributed except according to those terms. //! Mocked types for client-side and server-side APIs. +use std::cell::RefCell; use std::io::{self, Read, Write}; -use std::fmt; +use std::sync::{Once, ONCE_INIT}; +use std::{fmt, ptr, thread}; + +use log::{Metadata, Record}; use rand::{self, Rng, ThreadRng}; @@ -200,3 +204,66 @@ impl<'s, W: Write> Write for StdoutTee<'s, W> { self.stdout.flush() } } + +/// Suppress all logging inside `closure` unless there is a panic +pub fn log_on_panic() -> PanicLogger { + ::log::set_max_level(::log::LevelFilter::Trace); + + if ::log::logger() as *const ::log::Log != &LOGGER as *const ::log::Log { + ::log::set_logger(&LOGGER).unwrap(); + } + + PanicLogger(()) +} + +/// Struct that logs if the thread panics before it is dropped. +#[must_use] +pub struct PanicLogger(()); + +impl PanicLogger { + /// Clear the buffered logs + pub fn clear(self) { + drop(self); + } +} + +impl Drop for PanicLogger { + fn drop(&mut self) { + LOG.with(|log| { + let mut log = log.borrow_mut(); + if thread::panicking() { + println!("log for failing test:\n{}", *log); + } + log.clear(); + }); + } +} + +static INIT_PANIC_LOGGER: Once = ONCE_INIT; + +thread_local! { + static LOG: RefCell = RefCell::new(String::new()) +} + +static LOGGER: ThreadLocalLogger = ThreadLocalLogger; + +struct ThreadLocalLogger; + +impl ::log::Log for ThreadLocalLogger { + fn enabled(&self, metadata: &Metadata) -> bool { + true // metadata.target().contains("multipart") + } + + fn log(&self, record: &Record) { + if self.enabled(record.metadata()) { + LOG.with(|log| { + // format the message before pushing it to `ptr` since the fmt + // impls of the record arguments could try to log as well + let msg = format!("{}:{}: {}\n", record.level(), record.target(), record.args()); + log.borrow_mut().push_str(&msg); + }); + } + } + + fn flush(&self) {} +} diff --git a/multipart/src/server/boundary.rs b/multipart/src/server/boundary.rs index 0d7cdac2b..0440d17df 100644 --- a/multipart/src/server/boundary.rs +++ b/multipart/src/server/boundary.rs @@ -9,12 +9,10 @@ use ::safemem; -use super::buf_redux::BufReader; +use super::buf_redux; use super::buf_redux::strategy::{LessThan, AtEndLessThan}; use super::twoway; -use log::LogLevel; - use std::cmp; use std::borrow::Borrow; @@ -25,6 +23,8 @@ use self::State::*; pub const MIN_BUF_SIZE: usize = 1024; +type BufReader = buf_redux::BufReader; + #[derive(Debug, PartialEq, Eq)] enum State { Searching, @@ -35,7 +35,7 @@ enum State { /// A struct implementing `Read` and `BufRead` that will yield bytes until it sees a given sequence. #[derive(Debug)] pub struct BoundaryReader { - source: BufReader, + source: BufReader, boundary: Vec, search_idx: usize, state: State, @@ -60,7 +60,7 @@ impl BoundaryReader where R: Read { } fn read_to_boundary(&mut self) -> io::Result<&[u8]> { - let buf = self.source.fill_buf()?; + let buf = fill_buf_min(&mut self.source, self.search_idx + self.boundary.len())?; trace!("Buf: {:?}", String::from_utf8_lossy(buf)); @@ -137,7 +137,7 @@ impl BoundaryReader where R: Read { let consume_amt = { let min_len = self.boundary.len() + 4; - let buf = self.source.fill_buf()?; + let buf = fill_buf_min(&mut self.source, min_len)?; if buf.len() < min_len { return Err(io::Error::new(io::ErrorKind::UnexpectedEof, @@ -223,6 +223,19 @@ impl BufRead for BoundaryReader where R: Read { } } +fn fill_buf_min(rdr: &mut R, min: usize) -> io::Result<&[u8]> { + let mut last_len = 0; + + while { + let buf = rdr.fill_buf()?; + let cont = buf.len() > last_len && buf.len() < min; + last_len = buf.len(); + cont + } { /* nop */ } + + rdr.fill_buf() +} + #[cfg(test)] mod test { use super::BoundaryReader; @@ -240,7 +253,8 @@ mod test { #[test] fn test_boundary() { - let _ = ::env_logger::init(); + ::mock::log_on_panic(); + debug!("Testing boundary (no split)"); let src = &mut TEST_VAL.as_bytes(); @@ -285,7 +299,8 @@ mod test { #[test] fn test_split_boundary() { - let _ = ::env_logger::init(); + let logger = ::mock::log_on_panic(); + debug!("Testing boundary (split)"); let mut buf = String::new(); @@ -299,6 +314,7 @@ mod test { test_boundary_reader(&mut reader, &mut buf); } + logger.clear(); } fn test_boundary_reader(reader: &mut BoundaryReader, buf: &mut String) { diff --git a/multipart/src/server/field.rs b/multipart/src/server/field.rs index cca789f6d..fae659c77 100644 --- a/multipart/src/server/field.rs +++ b/multipart/src/server/field.rs @@ -64,21 +64,35 @@ fn with_headers(r: &mut R, closure: F) -> Result Ret { const HEADER_LEN: usize = 4; - let (consume, ret) = { + let consume; + let ret; + + let mut last_len = 0; + + loop { + // this should return a larger buffer each time let buf = r.fill_buf()?; + // buffer has stopped growing + if buf.len() == last_len { + return Err(ParseHeaderError::TooLarge); + } + let mut raw_headers = [EMPTY_HEADER; HEADER_LEN]; match httparse::parse_headers(buf, &mut raw_headers)? { - Status::Partial => return Err(ParseHeaderError::TooLarge), - Status::Complete((consume, raw_headers)) => { + // read more and try again + Status::Partial => last_len = buf.len(), + Status::Complete((consume_, raw_headers)) => { let mut headers = [EMPTY_STR_HEADER; HEADER_LEN]; let headers = copy_headers(raw_headers, &mut headers)?; debug!("Parsed headers: {:?}", headers); - (consume, closure(headers)) + consume = consume_; + ret = closure(headers); + break; }, } - }; + } r.consume(consume); Ok(ret) @@ -303,7 +317,9 @@ impl Read for MultipartData { } } -/// Unlike `std::io::BufReader`, +/// In this implementation, `fill_buf()` can return more data with each call. +/// +/// Use `set_min_buf_size()` if you require a minimum buffer length. impl BufRead for MultipartData { fn fill_buf(&mut self) -> io::Result<&[u8]> { self.inner_mut().source_mut().fill_buf() From 3cf4a42929b1b606b903eed3605ff2855b5d1a40 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Thu, 8 Feb 2018 18:42:17 -0800 Subject: [PATCH 389/453] improve and document `log_on_panic()` --- multipart/src/mock.rs | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/multipart/src/mock.rs b/multipart/src/mock.rs index 0581b96c2..b7cd13922 100644 --- a/multipart/src/mock.rs +++ b/multipart/src/mock.rs @@ -5,7 +5,7 @@ // http://opensource.org/licenses/MIT>, at your option. This file may not be // copied, modified, or distributed except according to those terms. //! Mocked types for client-side and server-side APIs. -use std::cell::RefCell; +use std::cell::{Cell, RefCell}; use std::io::{self, Read, Write}; use std::sync::{Once, ONCE_INIT}; use std::{fmt, ptr, thread}; @@ -205,14 +205,21 @@ impl<'s, W: Write> Write for StdoutTee<'s, W> { } } -/// Suppress all logging inside `closure` unless there is a panic +/// Capture all logging on this thread during `PanicLogger`'s lifetime. +/// +/// When `PanicLogger` is dropped: if the thread is panicking, all captured logs are printed +/// to stdout, otherwise the captured logs are deleted. +/// +/// RFC: break this functionality out into its own crate? pub fn log_on_panic() -> PanicLogger { ::log::set_max_level(::log::LevelFilter::Trace); if ::log::logger() as *const ::log::Log != &LOGGER as *const ::log::Log { - ::log::set_logger(&LOGGER).unwrap(); + ::log::set_logger(&LOGGER).expect("failed to set logger for `log_on_panic()`"); } + LOG_ENABLED.with(|flag| flag.set(true)); + PanicLogger(()) } @@ -229,10 +236,11 @@ impl PanicLogger { impl Drop for PanicLogger { fn drop(&mut self) { + LOG_ENABLED.with(|flag| flag.set(false)); LOG.with(|log| { let mut log = log.borrow_mut(); if thread::panicking() { - println!("log for failing test:\n{}", *log); + println!("captured logs before panic:\n{}", *log); } log.clear(); }); @@ -242,7 +250,8 @@ impl Drop for PanicLogger { static INIT_PANIC_LOGGER: Once = ONCE_INIT; thread_local! { - static LOG: RefCell = RefCell::new(String::new()) + static LOG: RefCell = RefCell::new(String::new()); + static LOG_ENABLED: Cell = Cell::new(bool); } static LOGGER: ThreadLocalLogger = ThreadLocalLogger; @@ -251,7 +260,7 @@ struct ThreadLocalLogger; impl ::log::Log for ThreadLocalLogger { fn enabled(&self, metadata: &Metadata) -> bool { - true // metadata.target().contains("multipart") + LOG_ENABLED.with(Cell::get) } fn log(&self, record: &Record) { From e3089e62afd48000a55d4dd702ecf00af5e890ac Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Thu, 8 Feb 2018 18:43:18 -0800 Subject: [PATCH 390/453] improve and document `log_on_panic()` (2) --- multipart/src/mock.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/multipart/src/mock.rs b/multipart/src/mock.rs index b7cd13922..a77d3cded 100644 --- a/multipart/src/mock.rs +++ b/multipart/src/mock.rs @@ -210,7 +210,9 @@ impl<'s, W: Write> Write for StdoutTee<'s, W> { /// When `PanicLogger` is dropped: if the thread is panicking, all captured logs are printed /// to stdout, otherwise the captured logs are deleted. /// -/// RFC: break this functionality out into its own crate? +/// RFC: break this functionality out into its own crate? Could also support nested invocations +/// (where each instance of `PanicLogger` only prints the logs captured during its lifetime) +/// but that's currently unused here. pub fn log_on_panic() -> PanicLogger { ::log::set_max_level(::log::LevelFilter::Trace); From dc1eb04f95f7326d01d168f4e34ec85b12e40a3c Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Thu, 8 Feb 2018 20:10:32 -0800 Subject: [PATCH 391/453] fix typo --- multipart/src/mock.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/src/mock.rs b/multipart/src/mock.rs index a77d3cded..68705568a 100644 --- a/multipart/src/mock.rs +++ b/multipart/src/mock.rs @@ -253,7 +253,7 @@ static INIT_PANIC_LOGGER: Once = ONCE_INIT; thread_local! { static LOG: RefCell = RefCell::new(String::new()); - static LOG_ENABLED: Cell = Cell::new(bool); + static LOG_ENABLED: Cell = Cell::new(false); } static LOGGER: ThreadLocalLogger = ThreadLocalLogger; From 0b68d184c5219ab6d8889d8a9085a6b6b583c4b6 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Mon, 12 Feb 2018 18:26:45 -0800 Subject: [PATCH 392/453] provide an example for Rocket bumps to 0.14.1 to publish updated README --- multipart/Cargo.toml | 22 +++++---- multipart/README.md | 6 +++ multipart/examples/README.md | 30 ++++++++----- multipart/examples/rocket.rs | 80 +++++++++++++++++++++++++++++++++ multipart/examples/tiny_http.rs | 2 +- 5 files changed, 120 insertions(+), 20 deletions(-) create mode 100644 multipart/examples/rocket.rs diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 191f0f085..175e1bd63 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "multipart" -version = "0.14.0" +version = "0.14.1" authors = ["Austin Bonander "] @@ -40,6 +40,8 @@ nickel = { version = ">=0.10.1", optional = true } [dev-dependencies] env_logger = "0.4" +rocket = "0.3" +rocket_codegen = "0.3" [features] client = [] @@ -56,28 +58,32 @@ use_arc_str = [] [[example]] name = "hyper_client" -required-features = ["mock", "hyper"] +required-features = ["client", "mock", "hyper"] [[example]] name = "hyper_reqbuilder" -required-features = ["mock", "hyper"] +required-features = ["client", "mock", "hyper"] [[example]] name = "hyper_server" -required-features = ["mock", "hyper"] +required-features = ["mock", "hyper", "server"] [[example]] name = "iron" -required-features = ["mock", "iron"] +required-features = ["mock", "iron", "server"] [[example]] name = "iron_intercept" -required-features = ["mock", "iron"] +required-features = ["mock", "iron", "server"] [[example]] name = "nickel" -required-features = ["mock", "nickel"] +required-features = ["mock", "nickel", "server"] [[example]] name = "tiny_http" -required-features = ["mock", "tiny_http"] +required-features = ["mock", "tiny_http", "server"] + +[[example]] +name = "rocket" +required-features = ["mock", "nightly", "server"] diff --git a/multipart/README.md b/multipart/README.md index e58e23d91..fd617ac98 100644 --- a/multipart/README.md +++ b/multipart/README.md @@ -39,6 +39,12 @@ via the `tiny_http` feature. Provides server-side integration with `tiny_http::Request` via `multipart::server::Multipart`. +### [Rocket ![](https://img.shields.io/crates/v/rocket.svg)](https://crates.io/crates/rocket) + +Direct integration is not provided as the Rocket folks seem to want to handle `multipart/form-data` +behind the scenes which would supercede any integration with `multipart`. However, an example is available +showing how to use `multipart` on a Rocket server: [examples/rocket.rs](examples/rocket.rs) + ## âš¡ Powered By âš¡ ### [buf_redux ![](https://img.shields.io/crates/v/buf_redux.svg)](https://crates.io/crates/buf_redux) diff --git a/multipart/examples/README.md b/multipart/examples/README.md index 00294c9c5..9465a442a 100644 --- a/multipart/examples/README.md +++ b/multipart/examples/README.md @@ -11,7 +11,7 @@ Examples for the client-side integrations of `multipart`'s API. [`hyper_client`](hyper_client.rs) --------------------------------- -Author: [abonander][abonander] +Author: [abonander] This example showcases usage of `multipart` with the `hyper::client::Request` API. @@ -21,7 +21,7 @@ $ cargo run --example hyper_client [`hyper_reqbuilder`](hyper_reqbuilder.rs) ----------------------------------------- -Author: [abonander][abonander] +Author: [abonander] This example showcases usage of `multipart` with Hyper's new `Client` API, via the lazy-writing capabilities of `multipart::client::lazy`. @@ -34,7 +34,7 @@ $ cargo run --example hyper_reqbuilder [`hyper_server`](hyper_server.rs) --------------------------------- -Author: [Puhrez][puhrez] +Author: [Puhrez] This example shows how to use `multipart` with a [`hyper::Server`] (http://hyper.rs/) to intercept multipart requests. @@ -44,7 +44,7 @@ $ cargo run --example hyper_server [`iron`](iron.rs) ----------------- -Author: [White-Oak][white-oak] +Author: [White-Oak] This example shows how to use `multipart` with the [Iron web application framework](http://ironframework.io/), via `multipart`'s support for the `iron::Request` type. @@ -57,7 +57,7 @@ $ cargo run --features iron --example iron [`iron_intercept`](iron_intercept.rs) ------------------------------------- -Author: [abonander][abonander] +Author: [abonander] This example shows how to use `multipart`'s specialized `Intercept` middleware with Iron, which reads out all fields and files to local storage so they can be accessed arbitrarily. @@ -68,7 +68,7 @@ $ cargo run --features iron --example iron_intercept [`tiny_http`](tiny_http.rs) --------------------------- -Author: [White-Oak][white-oak] +Author: [White-Oak] This example shows how to use `multipart` with the [`tiny_http` crate](https://crates.io/crates/tiny_http), via `multipart`'s support for the `tiny_http::Request` type. @@ -78,7 +78,7 @@ $ cargo run --features tiny_http --example tiny_http [`hyper_server`](hyper_server.rs) --------------------------------- -Author: [Puhrez][puhrez] +Author: [Puhrez] This example shows how to use `multipart` with a [`hyper::Server`] (http://hyper.rs/) to intercept multipart requests. @@ -87,8 +87,8 @@ $ cargo run --example hyper_server ``` [`nickel`](nickel.rs) ---------------------------------- -Author: [iamsebastian][iamsebastian] +--------------------- +Author: [iamsebastian] This example shows how to use `multipart` to handle multipart uploads in [nickel.rs](https://nickel.rs). @@ -96,9 +96,17 @@ This example shows how to use `multipart` to handle multipart uploads in [nickel $ cargo run --example nickel --features nickel ``` +[Rocket](rocket.rs) +------------------- +Author: [abonander] + +This example shows how `multipart`'s server API can be used with [Rocket](https://rocket.rs) without +explicit support (the Rocket folks seem to want to handle `multipart/form-data` behind the scenes +but haven't gotten around to implementing it yet; this would supercede any integration from `multipart`). + [iamsebastian]: https://github.com/iamsebastian -[puhrez]: https://github.com/puhrez -[white-oak]: https://github.com/white-oak +[Puhrez]: https://github.com/puhrez +[White-Oak]: https://github.com/white-oak [abonander]: https://github.com/abonander diff --git a/multipart/examples/rocket.rs b/multipart/examples/rocket.rs new file mode 100644 index 000000000..9e6a5eee2 --- /dev/null +++ b/multipart/examples/rocket.rs @@ -0,0 +1,80 @@ +// Example usage with Rocket (https://rocket.rs) +// +// Direct integration is not provided at this time as it appears the Rocket folks would prefer +// to handle multipart requests behind the scenes. +#![feature(plugin)] +#![plugin(rocket_codegen)] + +extern crate multipart; +extern crate rocket; + +use multipart::mock::StdoutTee; +use multipart::server::Multipart; +use multipart::server::save::Entries; +use multipart::server::save::SaveResult::*; + +use rocket::{Data, Request}; +use rocket::http::{ContentType, Status}; +use rocket::response::Stream; +use rocket::response::status::Custom; + +use std::io::{self, Cursor, Write}; + +#[post("/upload", data = "")] +fn multipart_upload(cont_type: &ContentType, data: Data) -> Result>>, Custom> { + // this can be implemented as a request guard but it seems like just more boilerplate + if !cont_type.is_form_data() { + return Err(Custom( + Status::BadRequest, + "Content-Type not multipart/form-data".into() + )); + } + + let (_, boundary) = cont_type.params().find(|&(k, v)| k == "boundary").ok_or_else( + || Custom( + Status::BadRequest, + "`Content-Type: multipart/form-data` boundary param not provided".into() + ) + )?; + + match process_upload(boundary, data) { + Ok(resp) => Ok(Stream::from(Cursor::new(resp))), + Err(err) => Err(Custom(Status::InternalServerError, err.to_string())) + } +} + +fn process_upload(boundary: &str, data: Data) -> io::Result> { + let mut out = Vec::new(); + + // saves all fields, any field longer than 10kB goes to a temporary directory + // Entries could implement FromData though that would give zero control over + // how the files are saved + match Multipart::with_body(data.open(), boundary).save().temp() { + Full(entries) => process_entries(entries, &mut out)?, + Partial(partial, reason) => { + writeln!(out, "Request partially processed: {:?}", reason)?; + if let Some(field) = partial.partial { + writeln!(out, "Stopped on field: {:?}", field.source.headers)?; + } + + process_entries(partial.entries, &mut out)? + }, + Error(e) => return Err(e), + } + + Ok(out) +} + +fn process_entries(entries: Entries, out: &mut Vec) -> io::Result<()> { + { + let stdout = io::stdout(); + let tee = StdoutTee::new(&mut out, &stdout); + entries.write_debug(tee)?; + } + + writeln!(out, "Entries processed") +} + +fn main() { + rocket::ignite().mount("/", routes![multipart_upload]).launch(); +} diff --git a/multipart/examples/tiny_http.rs b/multipart/examples/tiny_http.rs index 2f61b149a..40176cfed 100644 --- a/multipart/examples/tiny_http.rs +++ b/multipart/examples/tiny_http.rs @@ -54,7 +54,7 @@ fn process_request(request: &mut Request) -> io::Result> { /// Processes saved entries from multipart request. /// Returns an OK response or an error. -fn process_entries<'a>(entries: Entries) -> io::Result> { +fn process_entries(entries: Entries) -> io::Result> { let mut data = Vec::new(); { From b6de30ef7f13c1742c086e5f5d99746120495b05 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Mon, 12 Feb 2018 18:29:59 -0800 Subject: [PATCH 393/453] fix Rocket example --- multipart/examples/rocket.rs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/multipart/examples/rocket.rs b/multipart/examples/rocket.rs index 9e6a5eee2..c3e884529 100644 --- a/multipart/examples/rocket.rs +++ b/multipart/examples/rocket.rs @@ -13,7 +13,7 @@ use multipart::server::Multipart; use multipart::server::save::Entries; use multipart::server::save::SaveResult::*; -use rocket::{Data, Request}; +use rocket::Data; use rocket::http::{ContentType, Status}; use rocket::response::Stream; use rocket::response::status::Custom; @@ -21,8 +21,10 @@ use rocket::response::status::Custom; use std::io::{self, Cursor, Write}; #[post("/upload", data = "")] +// signature requires the request to have a `Content-Type` fn multipart_upload(cont_type: &ContentType, data: Data) -> Result>>, Custom> { - // this can be implemented as a request guard but it seems like just more boilerplate + // this and the next check can be implemented as a request guard but it seems like just + // more boilerplate than necessary if !cont_type.is_form_data() { return Err(Custom( Status::BadRequest, @@ -30,7 +32,7 @@ fn multipart_upload(cont_type: &ContentType, data: Data) -> Result io::Result> { // saves all fields, any field longer than 10kB goes to a temporary directory // Entries could implement FromData though that would give zero control over - // how the files are saved + // how the files are saved; Multipart would be a good impl candidate though match Multipart::with_body(data.open(), boundary).save().temp() { Full(entries) => process_entries(entries, &mut out)?, Partial(partial, reason) => { @@ -65,7 +67,7 @@ fn process_upload(boundary: &str, data: Data) -> io::Result> { Ok(out) } -fn process_entries(entries: Entries, out: &mut Vec) -> io::Result<()> { +fn process_entries(entries: Entries, mut out: &mut Vec) -> io::Result<()> { { let stdout = io::stdout(); let tee = StdoutTee::new(&mut out, &stdout); From 04eead38e7deed885b0b095164c3616d4d16f290 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Mon, 12 Feb 2018 18:32:32 -0800 Subject: [PATCH 394/453] fix rocket example (2) --- multipart/examples/rocket.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/multipart/examples/rocket.rs b/multipart/examples/rocket.rs index c3e884529..fc518c9d5 100644 --- a/multipart/examples/rocket.rs +++ b/multipart/examples/rocket.rs @@ -67,6 +67,8 @@ fn process_upload(boundary: &str, data: Data) -> io::Result> { Ok(out) } +// having a streaming output would be nice; there's one for returning a `Read` impl +// but not one that you can `write()` to fn process_entries(entries: Entries, mut out: &mut Vec) -> io::Result<()> { { let stdout = io::stdout(); From 2c008d57d8d3e5d12e789b118df49ecea87d6eef Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Thu, 15 Feb 2018 14:03:37 -0800 Subject: [PATCH 395/453] fix Travis testing of Rocket example (doesn't strictly need a new release since I doubt anyone's building examples from a Cargo release) --- multipart/.travis.yml | 3 ++- multipart/Cargo.toml | 8 +++++--- multipart/examples/README.md | 3 +++ 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/multipart/.travis.yml b/multipart/.travis.yml index 5913dd928..4ce2032d2 100644 --- a/multipart/.travis.yml +++ b/multipart/.travis.yml @@ -24,7 +24,8 @@ env: script: - if [ ${TRAVIS_RUST_VERSION} = "nightly" ]; then cargo build --verbose $ARGS --features "nightly"; - cargo test --verbose $ARGS --features "nightly" -- --test-threads=1; + # test Rocket example as well + cargo test --verbose $ARGS --features "nightly,rocket,rocket_codegen" -- --test-threads=1; cargo doc --verbose $ARGS --features "nightly"; fi - cargo build --verbose $ARGS; diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 175e1bd63..6c1bbf816 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -38,10 +38,12 @@ iron = { version = ">=0.4,<0.6", optional = true } tiny_http = { version = "0.5", optional = true } nickel = { version = ">=0.10.1", optional = true } +# Only for Rocket example but dev-dependencies can't be optional +rocket = { version = "0.3", optional = true } +rocket_codegen = { version = "0.3", optional = true } + [dev-dependencies] env_logger = "0.4" -rocket = "0.3" -rocket_codegen = "0.3" [features] client = [] @@ -86,4 +88,4 @@ required-features = ["mock", "tiny_http", "server"] [[example]] name = "rocket" -required-features = ["mock", "nightly", "server"] +required-features = ["mock", "rocket", "rocket_codegen", "server"] diff --git a/multipart/examples/README.md b/multipart/examples/README.md index 9465a442a..5f0b6999b 100644 --- a/multipart/examples/README.md +++ b/multipart/examples/README.md @@ -104,6 +104,9 @@ This example shows how `multipart`'s server API can be used with [Rocket](https: explicit support (the Rocket folks seem to want to handle `multipart/form-data` behind the scenes but haven't gotten around to implementing it yet; this would supercede any integration from `multipart`). +``` +$ cargo run --example rocket --features "rocket,rocket_codegen" +``` [iamsebastian]: https://github.com/iamsebastian [Puhrez]: https://github.com/puhrez From a94ea74f1acb1f521ca92c491c62325b9b735ea3 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Thu, 1 Mar 2018 05:02:07 -0800 Subject: [PATCH 396/453] widen Iron to include 0.6, bump to 0.14.2 --- multipart/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 6c1bbf816..a3fff6d65 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "multipart" -version = "0.14.1" +version = "0.14.2" authors = ["Austin Bonander "] @@ -34,7 +34,7 @@ quick-error = { version = "1.2", optional = true } # Optional Integrations hyper = { version = ">=0.9, <0.11", optional = true, default-features = false } -iron = { version = ">=0.4,<0.6", optional = true } +iron = { version = ">=0.4,<0.7", optional = true } tiny_http = { version = "0.5", optional = true } nickel = { version = ">=0.10.1", optional = true } From 4635330022602264f66f98fbceb8266462fa9e8f Mon Sep 17 00:00:00 2001 From: Ossi Herrala Date: Fri, 11 May 2018 20:49:19 +0300 Subject: [PATCH 397/453] Remove unused uses --- multipart/examples/hyper_server.rs | 5 ++--- multipart/examples/iron.rs | 5 ++--- multipart/examples/nickel.rs | 5 ++--- multipart/examples/tiny_http.rs | 5 ++--- multipart/src/client/lazy.rs | 4 ++-- multipart/src/lib.rs | 2 -- multipart/src/local_test.rs | 2 +- multipart/src/mock.rs | 2 +- multipart/src/server/boundary.rs | 1 - multipart/src/server/field.rs | 8 +++----- multipart/src/server/mod.rs | 5 ----- multipart/src/server/save.rs | 2 -- 12 files changed, 15 insertions(+), 31 deletions(-) diff --git a/multipart/examples/hyper_server.rs b/multipart/examples/hyper_server.rs index dc27e382b..6823400c3 100644 --- a/multipart/examples/hyper_server.rs +++ b/multipart/examples/hyper_server.rs @@ -1,13 +1,12 @@ extern crate hyper; extern crate multipart; -use std::fs::File; -use std::io::{self, Read}; +use std::io; use hyper::server::{Handler, Server, Request, Response}; use hyper::status::StatusCode; use hyper::server::response::Response as HyperResponse; use multipart::server::hyper::{Switch, MultipartHandler, HyperRequest}; -use multipart::server::{Multipart, Entries, SaveResult, SavedField}; +use multipart::server::{Multipart, Entries, SaveResult}; use multipart::mock::StdoutTee; struct NonMultipart; diff --git a/multipart/examples/iron.rs b/multipart/examples/iron.rs index ee9a8049b..34061d935 100644 --- a/multipart/examples/iron.rs +++ b/multipart/examples/iron.rs @@ -3,10 +3,9 @@ extern crate iron; extern crate env_logger; -use std::fs::File; -use std::io::{self, Read, Write}; +use std::io::{self, Write}; use multipart::mock::StdoutTee; -use multipart::server::{Multipart, Entries, SaveResult, SavedField}; +use multipart::server::{Multipart, Entries, SaveResult}; use iron::prelude::*; use iron::status; diff --git a/multipart/examples/nickel.rs b/multipart/examples/nickel.rs index b789c04bf..4fbe732d0 100644 --- a/multipart/examples/nickel.rs +++ b/multipart/examples/nickel.rs @@ -1,9 +1,8 @@ extern crate multipart; extern crate nickel; -use std::fs::File; -use std::io::{self, Read, Write}; -use nickel::{Action, HttpRouter, MiddlewareResult, Nickel, NickelError, Request, Response}; +use std::io::{self, Write}; +use nickel::{Action, HttpRouter, MiddlewareResult, Nickel, Request, Response}; use nickel::status::StatusCode; use multipart::server::nickel::MultipartBody; diff --git a/multipart/examples/tiny_http.rs b/multipart/examples/tiny_http.rs index 40176cfed..caef10914 100644 --- a/multipart/examples/tiny_http.rs +++ b/multipart/examples/tiny_http.rs @@ -1,9 +1,8 @@ extern crate tiny_http; extern crate multipart; -use std::fs::File; -use std::io::{self, Cursor, Read, Write}; -use multipart::server::{Multipart, Entries, SaveResult, SavedField}; +use std::io::{self, Cursor, Write}; +use multipart::server::{Multipart, Entries, SaveResult}; use multipart::mock::StdoutTee; use tiny_http::{Response, StatusCode, Request}; fn main() { diff --git a/multipart/src/client/lazy.rs b/multipart/src/client/lazy.rs index a33ce46a2..8e082980d 100644 --- a/multipart/src/client/lazy.rs +++ b/multipart/src/client/lazy.rs @@ -8,9 +8,9 @@ use std::path::{Path, PathBuf}; use std::io::prelude::*; use std::io::Cursor; -use std::{fmt, io, mem}; +use std::{fmt, io}; -use super::{HttpRequest, HttpStream, MultipartWriter}; +use super::{HttpRequest, HttpStream}; macro_rules! try_lazy ( ($field:expr, $try:expr) => ( diff --git a/multipart/src/lib.rs b/multipart/src/lib.rs index cde8cdef4..da7fa5c66 100644 --- a/multipart/src/lib.rs +++ b/multipart/src/lib.rs @@ -45,9 +45,7 @@ #[macro_use] extern crate log; -#[cfg_attr(test, macro_use)] extern crate mime; - extern crate mime_guess; extern crate rand; extern crate tempdir; diff --git a/multipart/src/local_test.rs b/multipart/src/local_test.rs index 0bf3d1430..b23b9225f 100644 --- a/multipart/src/local_test.rs +++ b/multipart/src/local_test.rs @@ -6,7 +6,7 @@ // copied, modified, or distributed except according to those terms. use mock::{ClientRequest, HttpBuffer}; -use server::{MultipartField, MultipartData, ReadEntry, FieldHeaders}; +use server::{MultipartField, ReadEntry, FieldHeaders}; use mime::{self, Mime}; diff --git a/multipart/src/mock.rs b/multipart/src/mock.rs index 68705568a..dc1b9eb3f 100644 --- a/multipart/src/mock.rs +++ b/multipart/src/mock.rs @@ -8,7 +8,7 @@ use std::cell::{Cell, RefCell}; use std::io::{self, Read, Write}; use std::sync::{Once, ONCE_INIT}; -use std::{fmt, ptr, thread}; +use std::{fmt, thread}; use log::{Metadata, Record}; diff --git a/multipart/src/server/boundary.rs b/multipart/src/server/boundary.rs index 0440d17df..e26ab1a2b 100644 --- a/multipart/src/server/boundary.rs +++ b/multipart/src/server/boundary.rs @@ -242,7 +242,6 @@ mod test { use std::io; use std::io::prelude::*; - use std::slice; const BOUNDARY: &'static str = "boundary"; const TEST_VAL: &'static str = "--boundary\r\n\ diff --git a/multipart/src/server/field.rs b/multipart/src/server/field.rs index fae659c77..03083699d 100644 --- a/multipart/src/server/field.rs +++ b/multipart/src/server/field.rs @@ -11,10 +11,8 @@ use mime::{self, Mime, FromStrError as MimeParseErr}; use quick_error::ResultExt; use std::error::Error; -use std::io::{self, Read, BufRead, Write}; -use std::ops::Deref; -use std::path::{Path, PathBuf}; -use std::{str, fmt, error}; +use std::io::{self, Read, BufRead}; +use std::{str, fmt}; use std::ascii::AsciiExt; @@ -22,7 +20,7 @@ use super::httparse::{self, EMPTY_HEADER, Header, Status, Error as HttparseError use self::ReadEntryResult::*; -use super::save::{SaveBuilder, SavedField}; +use super::save::SaveBuilder; use super::ArcStr; diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index 284df9b4c..fb4cfb8b7 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -17,12 +17,9 @@ extern crate twoway; use std::borrow::Borrow; use std::io::prelude::*; -use std::path::Path; use std::sync::Arc; use std::io; -use tempdir::TempDir; - use self::boundary::BoundaryReader; use self::field::PrivReadEntry; @@ -33,8 +30,6 @@ use self::save::SaveBuilder; pub use self::save::{Entries, SaveResult, SavedField}; -use self::save::EntriesSaveResult; - /// Default typedef for shared strings. /// /// Enable the `use_arc_str` feature to use `Arc` instead, which saves an indirection but diff --git a/multipart/src/server/save.rs b/multipart/src/server/save.rs index 3283c6d52..351d72a29 100644 --- a/multipart/src/server/save.rs +++ b/multipart/src/server/save.rs @@ -6,8 +6,6 @@ // copied, modified, or distributed except according to those terms. //! Utilities for saving request entries to the filesystem. -use mime::Mime; - pub use server::buf_redux::BufReader; pub use tempdir::TempDir; From a1dc4432b4f07ace59b21b0a49b01a8d2f9cb804 Mon Sep 17 00:00:00 2001 From: Ossi Herrala Date: Fri, 11 May 2018 20:58:58 +0300 Subject: [PATCH 398/453] Add allow unused_imports and deprecated before import of AsciiExt trait --- multipart/src/server/field.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/multipart/src/server/field.rs b/multipart/src/server/field.rs index 03083699d..a78003f34 100644 --- a/multipart/src/server/field.rs +++ b/multipart/src/server/field.rs @@ -14,6 +14,9 @@ use std::error::Error; use std::io::{self, Read, BufRead}; use std::{str, fmt}; +// The AsciiExt import is needed for Rust older than 1.23.0. These two lines can +// be removed when supporting older Rust is no longer needed. +#[allow(deprecated, unused_imports)] use std::ascii::AsciiExt; use super::httparse::{self, EMPTY_HEADER, Header, Status, Error as HttparseError}; From bd0e9a1966191056e4ce366a9b00dff05b17db7e Mon Sep 17 00:00:00 2001 From: Ossi Herrala Date: Fri, 11 May 2018 21:41:10 +0300 Subject: [PATCH 399/453] Update dependencies --- multipart/Cargo.toml | 14 +++++++------- multipart/examples/iron.rs | 2 +- multipart/src/server/boundary.rs | 22 ++++++++-------------- 3 files changed, 16 insertions(+), 22 deletions(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index a3fff6d65..13b775580 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -20,14 +20,14 @@ readme = "README.md" [dependencies] log = "0.4" mime = "0.3" -mime_guess = "2.0.0-alpha.3" -rand = "0.3" -safemem = { version = "0.2", optional = true } -tempdir = ">=0.3.4" +mime_guess = "2.0.0-alpha.4" +rand = "0.4" +safemem = { version = "0.3", optional = true } +tempdir = "0.3" clippy = { version = ">=0.0, <0.1", optional = true} #Server Dependencies -buf_redux = { version = "0.6", optional = true } +buf_redux = { version = "0.7", optional = true } httparse = { version = "1.2", optional = true } twoway = { version = "0.1", optional = true } quick-error = { version = "1.2", optional = true } @@ -35,7 +35,7 @@ quick-error = { version = "1.2", optional = true } # Optional Integrations hyper = { version = ">=0.9, <0.11", optional = true, default-features = false } iron = { version = ">=0.4,<0.7", optional = true } -tiny_http = { version = "0.5", optional = true } +tiny_http = { version = "0.6", optional = true } nickel = { version = ">=0.10.1", optional = true } # Only for Rocket example but dev-dependencies can't be optional @@ -43,7 +43,7 @@ rocket = { version = "0.3", optional = true } rocket_codegen = { version = "0.3", optional = true } [dev-dependencies] -env_logger = "0.4" +env_logger = "0.5" [features] client = [] diff --git a/multipart/examples/iron.rs b/multipart/examples/iron.rs index 34061d935..effeef68e 100644 --- a/multipart/examples/iron.rs +++ b/multipart/examples/iron.rs @@ -10,7 +10,7 @@ use iron::prelude::*; use iron::status; fn main() { - env_logger::init().unwrap(); + env_logger::init(); Iron::new(process_request).http("localhost:80").expect("Could not bind localhost:80"); } diff --git a/multipart/src/server/boundary.rs b/multipart/src/server/boundary.rs index e26ab1a2b..6e6c01ae5 100644 --- a/multipart/src/server/boundary.rs +++ b/multipart/src/server/boundary.rs @@ -9,8 +9,8 @@ use ::safemem; -use super::buf_redux; -use super::buf_redux::strategy::{LessThan, AtEndLessThan}; +use super::buf_redux::BufReader; +use super::buf_redux::policy::MinBuffered; use super::twoway; use std::cmp; @@ -23,8 +23,6 @@ use self::State::*; pub const MIN_BUF_SIZE: usize = 1024; -type BufReader = buf_redux::BufReader; - #[derive(Debug, PartialEq, Eq)] enum State { Searching, @@ -35,7 +33,7 @@ enum State { /// A struct implementing `Read` and `BufRead` that will yield bytes until it sees a given sequence. #[derive(Debug)] pub struct BoundaryReader { - source: BufReader, + source: BufReader, boundary: Vec, search_idx: usize, state: State, @@ -46,13 +44,10 @@ impl BoundaryReader where R: Read { pub fn from_reader>>(reader: R, boundary: B) -> BoundaryReader { let mut boundary = boundary.into(); safemem::prepend(b"--", &mut boundary); + let source = BufReader::new(reader).set_policy(MinBuffered(MIN_BUF_SIZE)); BoundaryReader { - source: BufReader::with_strategies( - reader, - LessThan(MIN_BUF_SIZE), - AtEndLessThan(MIN_BUF_SIZE), - ), + source, boundary, search_idx: 0, state: Searching, @@ -114,8 +109,7 @@ impl BoundaryReader where R: Read { // ensure the minimum buf size is at least enough to find a boundary with some extra let min_buf_size = cmp::max(self.boundary.len() * 2, min_buf_size); - self.source.read_strategy_mut().0 = min_buf_size; - self.source.move_strategy_mut().0 = min_buf_size; + self.source.policy_mut().0 = min_buf_size; } #[doc(hidden)] @@ -163,13 +157,13 @@ impl BoundaryReader where R: Read { trace!("Consuming {} bytes, remaining buf: {:?}", consume_amt, - String::from_utf8_lossy(self.source.get_buf())); + String::from_utf8_lossy(self.source.buffer())); self.source.consume(consume_amt); self.search_idx = 0; trace!("Consumed boundary (state: {:?}), remaining buf: {:?}", self.state, - String::from_utf8_lossy(self.source.get_buf())); + String::from_utf8_lossy(self.source.buffer())); Ok(self.state == AtEnd) } From 21b848a8d90e5e1ed1fae936a106e16d37a8cd6a Mon Sep 17 00:00:00 2001 From: Ossi Herrala Date: Fri, 11 May 2018 22:22:37 +0300 Subject: [PATCH 400/453] Fix compiler and Clippy nags --- multipart/examples/hyper_server.rs | 2 +- multipart/examples/nickel.rs | 2 +- multipart/src/client/lazy.rs | 18 ++++--------- multipart/src/client/mod.rs | 6 +---- multipart/src/client/sized.rs | 2 +- multipart/src/local_test.rs | 6 ++--- multipart/src/mock.rs | 19 ++++++-------- multipart/src/server/boundary.rs | 4 +-- multipart/src/server/field.rs | 41 +++++++++++++----------------- multipart/src/server/hyper.rs | 5 +--- multipart/src/server/iron.rs | 12 ++++----- multipart/src/server/save.rs | 18 +++++++------ multipart/src/server/tiny_http.rs | 2 +- 13 files changed, 57 insertions(+), 80 deletions(-) diff --git a/multipart/examples/hyper_server.rs b/multipart/examples/hyper_server.rs index 6823400c3..56ebc8a88 100644 --- a/multipart/examples/hyper_server.rs +++ b/multipart/examples/hyper_server.rs @@ -19,7 +19,7 @@ impl Handler for NonMultipart { struct EchoMultipart; impl MultipartHandler for EchoMultipart { - fn handle_multipart(&self, mut multipart: Multipart, mut res: HyperResponse) { + fn handle_multipart(&self, mut multipart: Multipart, res: HyperResponse) { match multipart.save().temp() { SaveResult::Full(entries) => process_entries(res, entries).unwrap(), SaveResult::Partial(entries, error) => { diff --git a/multipart/examples/nickel.rs b/multipart/examples/nickel.rs index 4fbe732d0..71ec8ff3c 100644 --- a/multipart/examples/nickel.rs +++ b/multipart/examples/nickel.rs @@ -40,7 +40,7 @@ fn process_entries<'mw>(res: Response<'mw>, entries: Entries) -> MiddlewareResul let stdout = io::stdout(); let mut res = res.start()?; if let Err(e) = entries.write_debug(StdoutTee::new(&mut res, &stdout)) { - writeln!(res, "Error while reading entries: {}", e); + writeln!(res, "Error while reading entries: {}", e).expect("writeln"); } Ok(Action::Halt(res)) diff --git a/multipart/src/client/lazy.rs b/multipart/src/client/lazy.rs index 8e082980d..f3b708e94 100644 --- a/multipart/src/client/lazy.rs +++ b/multipart/src/client/lazy.rs @@ -265,7 +265,7 @@ impl<'d> PreparedFields<'d> { use_len = false; streams.push( - PreparedField::from_stream(&field.name, &boundary, stream.content_type, + PreparedField::from_stream(&field.name, &boundary, &stream.content_type, stream.filename.as_ref().map(|f| &**f), stream.stream)); }, @@ -283,7 +283,7 @@ impl<'d> PreparedFields<'d> { Ok(PreparedFields { text_data: Cursor::new(text_data), - streams: streams, + streams, end_boundary: Cursor::new(boundary), content_len: if use_len { Some(content_len) } else { None } , }) @@ -347,14 +347,14 @@ impl<'d> PreparedField<'d> { let file = try_lazy!(name, File::open(path)); let content_len = try_lazy!(name, file.metadata()).len(); - let stream = Self::from_stream(&name, boundary, content_type, filename, Box::new(file)); + let stream = Self::from_stream(&name, boundary, &content_type, filename, Box::new(file)); let content_len = content_len + (stream.header.get_ref().len() as u64); Ok((stream, content_len)) } - fn from_stream(name: &str, boundary: &str, content_type: Mime, filename: Option<&str>, stream: Box) -> Self { + fn from_stream(name: &str, boundary: &str, content_type: &Mime, filename: Option<&str>, stream: Box) -> Self { let mut header = Vec::new(); write!(header, "{}\r\nContent-Disposition: form-data; name=\"{}\"", @@ -368,7 +368,7 @@ impl<'d> PreparedField<'d> { PreparedField { header: Cursor::new(header), - stream: stream + stream, } } } @@ -394,14 +394,6 @@ impl<'d> fmt::Debug for PreparedField<'d> { } } -struct CowStrAsRef<'d>(Cow<'d, str>); - -impl<'d> AsRef<[u8]> for CowStrAsRef<'d> { - fn as_ref(&self) -> &[u8] { - self.0.as_bytes() - } -} - /// Conversion trait necessary for `Multipart::add_file()` to accept borrowed or owned strings /// and borrowed or owned paths pub trait IntoCowPath<'a> { diff --git a/multipart/src/client/mod.rs b/multipart/src/client/mod.rs index afee0c048..40d535130 100644 --- a/multipart/src/client/mod.rs +++ b/multipart/src/client/mod.rs @@ -198,7 +198,7 @@ struct MultipartWriter<'a, W> { impl<'a, W: Write> MultipartWriter<'a, W> { fn new>>(inner: W, boundary: B) -> Self { MultipartWriter { - inner: inner, + inner, boundary: boundary.into(), data_written: false, } @@ -251,10 +251,6 @@ impl<'a, W: Write> MultipartWriter<'a, W> { } } - fn inner_mut(&mut self) -> &mut W { - &mut self.inner - } - fn finish(mut self) -> io::Result { if self.data_written { // Write two hyphens after the last boundary occurrence. diff --git a/multipart/src/client/sized.rs b/multipart/src/client/sized.rs index e7caf6b3f..75a3945c0 100644 --- a/multipart/src/client/sized.rs +++ b/multipart/src/client/sized.rs @@ -85,6 +85,6 @@ where ::Error: From { let mut req = self.inner.open_stream()?; io::copy(&mut &self.buffer[..], &mut req)?; - req.finish().into() + req.finish() } } diff --git a/multipart/src/local_test.rs b/multipart/src/local_test.rs index b23b9225f..3b648738d 100644 --- a/multipart/src/local_test.rs +++ b/multipart/src/local_test.rs @@ -329,7 +329,7 @@ fn test_client(test_fields: &TestFields) -> HttpBuffer { |(name, files)| files.iter().map(move |file| (name, file)) ); - let mut test_texts = test_fields.texts.iter().flat_map( + let test_texts = test_fields.texts.iter().flat_map( |(name, texts)| texts.iter().map(move |text| (name, text)) ); @@ -363,7 +363,7 @@ fn test_client_lazy(test_fields: &TestFields) -> HttpBuffer { |(name, files)| files.iter().map(move |file| (name, file)) ); - let mut test_texts = test_fields.texts.iter().flat_map( + let test_texts = test_fields.texts.iter().flat_map( |(name, texts)| texts.iter().map(move |text| (name, text)) ); @@ -405,7 +405,7 @@ fn test_server(buf: HttpBuffer, fields: &mut TestFields) { let mut multipart = Multipart::from_request(server_buf) .unwrap_or_else(|_| panic!("Buffer should be multipart!")); - while let Some(mut field) = multipart.read_entry_mut().unwrap_opt() { + while let Some(field) = multipart.read_entry_mut().unwrap_opt() { fields.check_field(field); } } diff --git a/multipart/src/mock.rs b/multipart/src/mock.rs index dc1b9eb3f..57dad3cfe 100644 --- a/multipart/src/mock.rs +++ b/multipart/src/mock.rs @@ -7,7 +7,6 @@ //! Mocked types for client-side and server-side APIs. use std::cell::{Cell, RefCell}; use std::io::{self, Read, Write}; -use std::sync::{Once, ONCE_INIT}; use std::{fmt, thread}; use log::{Metadata, Record}; @@ -67,9 +66,9 @@ impl HttpBuffer { /// Wrap the given buffer with the given boundary and optional content-length. pub fn with_buf(buf: Vec, boundary: String, content_len: Option) -> Self { HttpBuffer { - buf: buf, - boundary: boundary, - content_len: content_len, + buf, + boundary, + content_len, rng: rand::thread_rng() } } @@ -144,8 +143,8 @@ impl<'a> ServerRequest<'a> { /// Assumes `content_len: None` pub fn new(data: &'a [u8], boundary: &'a str) -> Self { ServerRequest { - data: data, - boundary: boundary, + data, + boundary, content_len: None, rng: rand::thread_rng(), } @@ -195,12 +194,12 @@ impl<'s, W> StdoutTee<'s, W> { impl<'s, W: Write> Write for StdoutTee<'s, W> { fn write(&mut self, buf: &[u8]) -> io::Result { - self.inner.write(buf)?; + self.inner.write_all(buf)?; self.stdout.write(buf) } fn flush(&mut self) -> io::Result<()> { - self.inner.flush(); + self.inner.flush()?; self.stdout.flush() } } @@ -249,8 +248,6 @@ impl Drop for PanicLogger { } } -static INIT_PANIC_LOGGER: Once = ONCE_INIT; - thread_local! { static LOG: RefCell = RefCell::new(String::new()); static LOG_ENABLED: Cell = Cell::new(false); @@ -261,7 +258,7 @@ static LOGGER: ThreadLocalLogger = ThreadLocalLogger; struct ThreadLocalLogger; impl ::log::Log for ThreadLocalLogger { - fn enabled(&self, metadata: &Metadata) -> bool { + fn enabled(&self, _: &Metadata) -> bool { LOG_ENABLED.with(Cell::get) } diff --git a/multipart/src/server/boundary.rs b/multipart/src/server/boundary.rs index 6e6c01ae5..56f6b0dbe 100644 --- a/multipart/src/server/boundary.rs +++ b/multipart/src/server/boundary.rs @@ -92,7 +92,7 @@ impl BoundaryReader where R: Read { trace!("Two bytes before: {:?} ({:?}) (\"\\r\\n\": {:?})", String::from_utf8_lossy(two_bytes_before), two_bytes_before, b"\r\n"); - if two_bytes_before == &*b"\r\n" { + if two_bytes_before == *b"\r\n" { debug!("Subtract two!"); buf_len -= 2; } @@ -246,7 +246,7 @@ mod test { #[test] fn test_boundary() { - ::mock::log_on_panic(); + let _logger = ::mock::log_on_panic(); debug!("Testing boundary (no split)"); diff --git a/multipart/src/server/field.rs b/multipart/src/server/field.rs index a78003f34..ccbbce2d8 100644 --- a/multipart/src/server/field.rs +++ b/multipart/src/server/field.rs @@ -59,8 +59,6 @@ impl <'s, 'a: 's> fmt::Display for DisplayHeaders<'s, 'a> { } } -const MAX_ATTEMPTS: usize = 30; - fn with_headers(r: &mut R, closure: F) -> Result where R: BufRead, F: FnOnce(&[StrHeader]) -> Ret { const HEADER_LEN: usize = 4; @@ -272,11 +270,11 @@ pub struct MultipartData { inner: Option, } -const DATA_INNER_ERR: &'static str = "MultipartFile::inner taken and not replaced; this is likely \ - caused by a logic error in `multipart` or by resuming after \ - a previously caught panic.\nPlease open an issue with the \ - relevant backtrace and debug logs at \ - https://github.com/abonander/multipart"; +const DATA_INNER_ERR: &str = "MultipartFile::inner taken and not replaced; this is likely \ + caused by a logic error in `multipart` or by resuming after \ + a previously caught panic.\nPlease open an issue with the \ + relevant backtrace and debug logs at \ + https://github.com/abonander/multipart"; impl MultipartData where M: ReadEntry { /// Get a builder type which can save the field with or without a size limit. @@ -371,15 +369,17 @@ pub trait ReadEntry: PrivReadEntry + Sized { let field_headers: FieldHeaders = try_read_entry!(self; self.read_headers()); - field_headers.content_type.as_ref().map(|ct| if ct.type_() == mime::MULTIPART { - // fields of this type are sent by (supposedly) no known clients - // (https://tools.ietf.org/html/rfc7578#appendix-A) so I'd be fascinated - // to hear about any in the wild - info!("Found nested multipart field: {:?}:\r\n\ - Please report this client's User-Agent and any other available details \ - at https://github.com/abonander/multipart/issues/56", - field_headers); - }); + if let Some(ct) = field_headers.content_type.as_ref() { + if ct.type_() == mime::MULTIPART { + // fields of this type are sent by (supposedly) no known clients + // (https://tools.ietf.org/html/rfc7578#appendix-A) so I'd be fascinated + // to hear about any in the wild + info!("Found nested multipart field: {:?}:\r\n\ + Please report this client's User-Agent and any other available details \ + at https://github.com/abonander/multipart/issues/56", + field_headers); + } + } Entry( MultipartField { @@ -504,16 +504,9 @@ impl ReadEntryResult { Error(_, err) => panic!("{}: {:?}", msg, err), } } - - fn invalid_data(multipart: M, msg: String) -> Self { - ReadEntryResult::Error ( - multipart, - io::Error::new(io::ErrorKind::InvalidData, msg), - ) - } } -const GENERIC_PARSE_ERR: &'static str = "an error occurred while parsing field headers"; +const GENERIC_PARSE_ERR: &str = "an error occurred while parsing field headers"; quick_error! { #[derive(Debug)] diff --git a/multipart/src/server/hyper.rs b/multipart/src/server/hyper.rs index e1697395a..57022dc6d 100644 --- a/multipart/src/server/hyper.rs +++ b/multipart/src/server/hyper.rs @@ -36,10 +36,7 @@ impl Switch where H: Handler, M: MultipartHandler { /// Create a new `Switch` instance where /// `normal` handles normal Hyper requests and `multipart` handles Multipart requests pub fn new(normal: H, multipart: M) -> Switch { - Switch { - normal: normal, - multipart: multipart, - } + Switch { normal, multipart } } } diff --git a/multipart/src/server/iron.rs b/multipart/src/server/iron.rs index 43d050812..898ab5dd6 100644 --- a/multipart/src/server/iron.rs +++ b/multipart/src/server/iron.rs @@ -57,7 +57,7 @@ pub const DEFAULT_FILE_COUNT_LIMIT: u32 = 16; /// fn main() { /// let mut chain = Chain::new(|req: &mut Request| if let Some(entries) = /// req.extensions.get::() { -/// +/// /// Ok(Response::with(format!("{:?}", entries))) /// } else { /// Ok(Response::with("Not a multipart request")) @@ -74,14 +74,14 @@ pub const DEFAULT_FILE_COUNT_LIMIT: u32 = 16; pub struct Intercept { /// The parent directory for all temporary directories created by this middleware. /// Will be created if it doesn't exist (lazy). - /// + /// /// If omitted, uses the OS temporary directory. /// /// Default value: `None`. pub temp_dir_path: Option, /// The size limit of uploaded files, in bytes. /// - /// Files which exceed this size will be rejected. + /// Files which exceed this size will be rejected. /// See the `limit_behavior` field for more info. /// /// Default value: [`DEFAULT_FILE_SIZE_LIMIT`](constant.default_file_size_limit.html) @@ -97,7 +97,7 @@ pub struct Intercept { pub limit_behavior: LimitBehavior, } -impl Intercept { +impl Intercept { /// Set the `temp_dir_path` for this middleware. pub fn temp_dir_path>(self, path: P) -> Self { Intercept { temp_dir_path: Some(path.into()), .. self } @@ -182,8 +182,8 @@ impl Intercept { type IronMultipart<'r, 'a, 'b> = Multipart<&'r mut IronBody<'a, 'b>>; -const EXPECT_PARTIAL_FILE: &'static str = "File size limit hit but the offending \ - file was not available; this is a bug."; +const EXPECT_PARTIAL_FILE: &str = "File size limit hit but the offending \ + file was not available; this is a bug."; impl Default for Intercept { fn default() -> Self { diff --git a/multipart/src/server/save.rs b/multipart/src/server/save.rs index 351d72a29..0c0a03748 100644 --- a/multipart/src/server/save.rs +++ b/multipart/src/server/save.rs @@ -272,7 +272,7 @@ impl SaveBuilder where M: ReadEntry { try_start!(create_dir_all(&dir)); - self.with_entries(Entries::new(SaveDir::Perm(dir.into()))) + self.with_entries(Entries::new(SaveDir::Perm(dir))) } /// Commence the save operation using the existing `Entries` instance. @@ -284,7 +284,7 @@ impl SaveBuilder where M: ReadEntry { /// /// Note that `PartialReason::CountLimit` will still be returned if the number of fields /// reaches `u32::MAX`, but this would be an extremely degenerate case. - pub fn with_entries(mut self, mut entries: Entries) -> EntriesSaveResult { + pub fn with_entries(self, mut entries: Entries) -> EntriesSaveResult { let SaveBuilder { savable, open_opts, count_limit, size_limit, memory_threshold, text_policy @@ -458,10 +458,12 @@ impl<'m, M: 'm> SaveBuilder<&'m mut MultipartData> where MultipartData: Bu match str::from_utf8(buf) { Ok(s) => { string.push_str(s); Full(buf.len()) }, // buffer should always be bigger - Err(e) => if buf.len() < 4 { return Partial(0, e.into())} else { - string.push_str(str::from_utf8(&buf[..e.valid_up_to()]).unwrap()); - Full(e.valid_up_to()) - } + Err(e) => if buf.len() < 4 { + Partial(0, e.into()) + } else { + string.push_str(str::from_utf8(&buf[..e.valid_up_to()]).unwrap()); + Full(e.valid_up_to()) + } } }, 0); @@ -884,7 +886,7 @@ pub enum SaveResult { } /// Shorthand result for methods that return `Entries` -pub type EntriesSaveResult = SaveResult>; +pub type EntriesSaveResult = SaveResult>; /// Shorthand result for methods that return `FieldData`s. /// @@ -966,7 +968,7 @@ fn create_dir_all(path: &Path) -> io::Result<()> { } } -fn try_copy_limited SaveResult>(mut src: R, mut with_buf: Wb, limit: u64) -> SaveResult { +fn try_copy_limited SaveResult>(src: R, mut with_buf: Wb, limit: u64) -> SaveResult { let mut copied = 0u64; try_read_buf(src, |buf| { let new_copied = copied.saturating_add(buf.len() as u64); diff --git a/multipart/src/server/tiny_http.rs b/multipart/src/server/tiny_http.rs index 175c5bc40..b2eaa8594 100644 --- a/multipart/src/server/tiny_http.rs +++ b/multipart/src/server/tiny_http.rs @@ -14,7 +14,7 @@ impl<'r> HttpRequest for &'r mut TinyHttpRequest { type Body = &'r mut Read; fn multipart_boundary(&self) -> Option<&str> { - const BOUNDARY: &'static str = "boundary="; + const BOUNDARY: &str = "boundary="; let content_type = try_opt!(self.headers().iter().find(|header| header.field.equiv("Content-Type"))).value.as_str(); let start = try_opt!(content_type.find(BOUNDARY)) + BOUNDARY.len(); From 4915acfcdf738c9c01c447cd8f5f4be1391f65d5 Mon Sep 17 00:00:00 2001 From: Ossi Herrala Date: Fri, 11 May 2018 22:28:56 +0300 Subject: [PATCH 401/453] Bump minimum rust version to 1.22.1 because of dependencies * unicode-normalization 0.1.7 seems to require at least Rust 1.21.0 * slice-deque 0.1.8 seems to require at least Rust 1.22.1 --- multipart/.travis.yml | 2 +- multipart/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/multipart/.travis.yml b/multipart/.travis.yml index 4ce2032d2..a23102d33 100644 --- a/multipart/.travis.yml +++ b/multipart/.travis.yml @@ -5,7 +5,7 @@ branches: except: - fuzzing rust: - - 1.17.0 + - 1.22.1 - stable - beta - nightly diff --git a/multipart/README.md b/multipart/README.md index fd617ac98..b7243a3dd 100644 --- a/multipart/README.md +++ b/multipart/README.md @@ -5,7 +5,7 @@ Client- and server-side abstractions for HTTP file uploads (POST requests with Supports several different (**sync**hronous API) HTTP crates. **Async**hronous API support will be provided by [multipart-async]. -Minimum supported Rust version: 1.17.0 +Minimum supported Rust version: 1.22.1 ### [Documentation](http://docs.rs/multipart/) From db09659dd010b5d9ef0f14ed73d9ce97e17f7f30 Mon Sep 17 00:00:00 2001 From: Ossi Herrala Date: Fri, 11 May 2018 22:37:37 +0300 Subject: [PATCH 402/453] Try to enable OS X builds again --- multipart/.travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/multipart/.travis.yml b/multipart/.travis.yml index a23102d33..b83d6ca65 100644 --- a/multipart/.travis.yml +++ b/multipart/.travis.yml @@ -11,8 +11,7 @@ rust: - nightly os: - linux - # openssl is broken on OS X again - # - osx + - osx env: global: - RUST_LOG=multipart=trace RUST_BACKTRACE=1 ARGS= From 3b6134133c27811b2c8960ae5144364cd88ae3cf Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Thu, 7 Jun 2018 08:59:52 -0700 Subject: [PATCH 403/453] try to fix .travis.yml --- multipart/.travis.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/multipart/.travis.yml b/multipart/.travis.yml index b83d6ca65..bd1922f3c 100644 --- a/multipart/.travis.yml +++ b/multipart/.travis.yml @@ -1,6 +1,5 @@ language: rust cache: cargo -sudo: false branches: except: - fuzzing @@ -22,10 +21,10 @@ env: - ARGS+='--features "use-arc-str"' script: - if [ ${TRAVIS_RUST_VERSION} = "nightly" ]; then - cargo build --verbose $ARGS --features "nightly"; - # test Rocket example as well - cargo test --verbose $ARGS --features "nightly,rocket,rocket_codegen" -- --test-threads=1; - cargo doc --verbose $ARGS --features "nightly"; + cargo build --verbose $ARGS --features "nightly"; + # test Rocket example as well + cargo test --verbose $ARGS --features "nightly,rocket,rocket_codegen" -- --test-threads=1; + cargo doc --verbose $ARGS --features "nightly"; fi - cargo build --verbose $ARGS; - cargo test --verbose $ARGS -- --test-threads=1; From e8c48d67cf5288ab4cf3790f2d7d5b2a7f13214d Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Thu, 7 Jun 2018 09:02:05 -0700 Subject: [PATCH 404/453] .travis.yml: fix syntax --- multipart/.travis.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/multipart/.travis.yml b/multipart/.travis.yml index bd1922f3c..1f6409adf 100644 --- a/multipart/.travis.yml +++ b/multipart/.travis.yml @@ -20,11 +20,12 @@ env: - ARGS+='--no-default-features --features "nickel"' - ARGS+='--features "use-arc-str"' script: - - if [ ${TRAVIS_RUST_VERSION} = "nightly" ]; then - cargo build --verbose $ARGS --features "nightly"; - # test Rocket example as well - cargo test --verbose $ARGS --features "nightly,rocket,rocket_codegen" -- --test-threads=1; - cargo doc --verbose $ARGS --features "nightly"; + - | + if [ ${TRAVIS_RUST_VERSION} = "nightly" ]; then + cargo build --verbose $ARGS --features "nightly"; + # test Rocket example as well + cargo test --verbose $ARGS --features "nightly,rocket,rocket_codegen" -- --test-threads=1; + cargo doc --verbose $ARGS --features "nightly"; fi - cargo build --verbose $ARGS; - cargo test --verbose $ARGS -- --test-threads=1; From bc260ed43a1b4ec89fe881769dda27a1895edd56 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Thu, 5 Jul 2018 18:13:15 -0700 Subject: [PATCH 405/453] Downgrade `mime` to 0.2 to be compatible with Hyper --- multipart/Cargo.toml | 4 ++-- multipart/src/lib.rs | 1 + multipart/src/local_test.rs | 8 ++++---- multipart/src/server/field.rs | 18 ++++++++---------- 4 files changed, 15 insertions(+), 16 deletions(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 13b775580..23849d549 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -19,8 +19,8 @@ readme = "README.md" [dependencies] log = "0.4" -mime = "0.3" -mime_guess = "2.0.0-alpha.4" +mime = "0.2" +mime_guess = "1.8" rand = "0.4" safemem = { version = "0.3", optional = true } tempdir = "0.3" diff --git a/multipart/src/lib.rs b/multipart/src/lib.rs index da7fa5c66..8f4eca6d8 100644 --- a/multipart/src/lib.rs +++ b/multipart/src/lib.rs @@ -45,6 +45,7 @@ #[macro_use] extern crate log; +#[macro_use] extern crate mime; extern crate mime_guess; extern crate rand; diff --git a/multipart/src/local_test.rs b/multipart/src/local_test.rs index 3b648738d..5dbf4bf20 100644 --- a/multipart/src/local_test.rs +++ b/multipart/src/local_test.rs @@ -145,7 +145,7 @@ impl FileEntry { ( FileEntry { - content_type: field.headers.content_type.unwrap_or(mime::APPLICATION_OCTET_STREAM), + content_type: field.headers.content_type.unwrap_or(mime!(Application/OctetStream)), filename: field.headers.filename, data: PrintHex(data), }, @@ -434,8 +434,8 @@ fn rand_mime() -> Mime { rand::thread_rng().choose(&[ // TODO: fill this out, preferably with variants that may be hard to parse // i.e. containing hyphens, mainly - mime::APPLICATION_OCTET_STREAM, - mime::TEXT_PLAIN, - mime::IMAGE_PNG, + mime!(Application/OctetStream), + mime!(Text/Plain), + mime!(Image/Png), ]).unwrap().clone() } diff --git a/multipart/src/server/field.rs b/multipart/src/server/field.rs index ccbbce2d8..0c25c638c 100644 --- a/multipart/src/server/field.rs +++ b/multipart/src/server/field.rs @@ -6,7 +6,7 @@ // copied, modified, or distributed except according to those terms. //! `multipart` field header parsing. -use mime::{self, Mime, FromStrError as MimeParseErr}; +use mime::{Mime, TopLevel, SubLevel}; use quick_error::ResultExt; @@ -199,7 +199,8 @@ fn parse_content_type(headers: &[StrHeader]) -> Result, ParseHeader if let Some(header) = find_header(headers, "Content-Type") { // Boundary parameter will be parsed into the `Mime` debug!("Found Content-Type: {:?}", header.val); - Ok(Some(header.val.parse::().context(header.val)?)) + Ok(Some(header.val.parse::() + .map_err(|_| ParseHeaderError::MimeError(header.val.into()))?)) } else { Ok(None) } @@ -230,7 +231,7 @@ impl MultipartField { /// /// Detecting character encodings by any means is (currently) beyond the scope of this crate. pub fn is_text(&self) -> bool { - self.headers.content_type.as_ref().map_or(true, |ct| ct.type_() == mime::TEXT) + self.headers.content_type.as_ref().map_or(true, |ct| ct.0 == TopLevel::Text) } /// Read the next entry in the request. @@ -370,7 +371,7 @@ pub trait ReadEntry: PrivReadEntry + Sized { let field_headers: FieldHeaders = try_read_entry!(self; self.read_headers()); if let Some(ct) = field_headers.content_type.as_ref() { - if ct.type_() == mime::MULTIPART { + if ct.0 == TopLevel::Multipart { // fields of this type are sent by (supposedly) no known clients // (https://tools.ietf.org/html/rfc7578#appendix-A) so I'd be fascinated // to hear about any in the wild @@ -527,12 +528,9 @@ quick_error! { cause(err) from() } - MimeError(err: MimeParseErr, cont_type: String) { - description(err.description()) - display("{}\nwhile parsing Content-Type value: {}", err.description(), cont_type) - cause(err) - context(cont_type: &'a str, err: MimeParseErr) -> - (err, cont_type.to_string()) + MimeError(cont_type: String) { + description("Failed to parse Content-Type") + display(this) -> ("{}: {}", this.description(), cont_type) } TooLarge { description("field headers section ridiculously long or missing trailing CRLF-CRLF") From d398232930c5a522d80328d3efff403ddb8e18c4 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sun, 8 Jul 2018 13:20:33 -0700 Subject: [PATCH 406/453] Crate version 0.15.0 --- multipart/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 23849d549..bcc057259 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "multipart" -version = "0.14.2" +version = "0.15.0" authors = ["Austin Bonander "] From 70e6a2882853a5a3d459e9248a47381fdd4a24c5 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Wed, 25 Jul 2018 23:06:41 -0700 Subject: [PATCH 407/453] fix reading of unterminated body closes abonander/multipart#104 --- multipart/src/server/boundary.rs | 72 +++++++++++++++++++++++++------- 1 file changed, 57 insertions(+), 15 deletions(-) diff --git a/multipart/src/server/boundary.rs b/multipart/src/server/boundary.rs index 56f6b0dbe..342eb4d47 100644 --- a/multipart/src/server/boundary.rs +++ b/multipart/src/server/boundary.rs @@ -55,7 +55,14 @@ impl BoundaryReader where R: Read { } fn read_to_boundary(&mut self) -> io::Result<&[u8]> { - let buf = fill_buf_min(&mut self.source, self.search_idx + self.boundary.len())?; + if self.state == AtEnd { return Ok(&[])} + + let buf = self.source.fill_buf()?; + + if self.search_idx == 0 && buf.len() < self.boundary.len() { + return Err(io::Error::new(io::ErrorKind::UnexpectedEof, + "unexpected end of request body")); + } trace!("Buf: {:?}", String::from_utf8_lossy(buf)); @@ -131,7 +138,7 @@ impl BoundaryReader where R: Read { let consume_amt = { let min_len = self.boundary.len() + 4; - let buf = fill_buf_min(&mut self.source, min_len)?; + let buf = self.source.fill_buf()?; if buf.len() < min_len { return Err(io::Error::new(io::ErrorKind::UnexpectedEof, @@ -217,19 +224,6 @@ impl BufRead for BoundaryReader where R: Read { } } -fn fill_buf_min(rdr: &mut R, min: usize) -> io::Result<&[u8]> { - let mut last_len = 0; - - while { - let buf = rdr.fill_buf()?; - let cont = buf.len() > last_len && buf.len() < min; - last_len = buf.len(); - cont - } { /* nop */ } - - rdr.fill_buf() -} - #[cfg(test)] mod test { use super::BoundaryReader; @@ -344,6 +338,8 @@ mod test { #[test] fn test_leading_crlf() { + let logger = ::mock::log_on_panic(); + let mut body: &[u8] = b"\r\n\r\n--boundary\r\n\ asdf1234\ \r\n\r\n--boundary--"; @@ -366,10 +362,14 @@ mod test { debug!("Read 2 (empty)"); let _ = reader.read_to_string(buf).unwrap(); assert_eq!(buf, ""); + + logger.clear() } #[test] fn test_trailing_crlf() { + let logger = ::mock::log_on_panic(); + let mut body: &[u8] = b"--boundary\r\n\ asdf1234\ \r\n\r\n--boundary\r\n\ @@ -407,11 +407,15 @@ mod test { debug!("Read 3 (empty)"); let _ = reader.read_to_string(buf).unwrap(); assert_eq!(buf, ""); + + logger.clear(); } // https://github.com/abonander/multipart/issues/93#issuecomment-343610587 #[test] fn test_trailing_lflf() { + let logger = ::mock::log_on_panic(); + let mut body: &[u8] = b"--boundary\r\n\ asdf1234\ \n\n\r\n--boundary\r\n\ @@ -448,6 +452,44 @@ mod test { debug!("Read 3 (empty)"); let _ = reader.read_to_string(buf).unwrap(); assert_eq!(buf, ""); + + logger.clear(); + } + + // https://github.com/abonander/multipart/issues/104 + #[test] + fn test_unterminated_body() { + let logger = ::mock::log_on_panic(); + + let mut body: &[u8] = b"--boundary\r\n\ + asdf1234\ + \n\n\r\n--boundary\r\n\ + hjkl5678"; + + let ref mut buf = String::new(); + let mut reader = BoundaryReader::from_reader(&mut body, BOUNDARY); + + debug!("Consume 1"); + reader.consume_boundary().unwrap(); + + debug!("Read 1"); + + // same as above + let buf1 = reader.read_to_boundary().unwrap().to_owned(); + let buf2 = reader.read_to_boundary().unwrap().to_owned(); + assert_eq!(buf1, buf2); + + let _ = reader.read_to_string(buf).unwrap(); + assert_eq!(buf, "asdf1234\n\n"); + buf.clear(); + + debug!("Consume 2"); + reader.consume_boundary().unwrap(); + + debug!("Read 2"); + let _ = reader.read_to_string(buf).unwrap_err(); + + logger.clear(); } #[cfg(feature = "bench")] From afb2cf6a2ea0b671e464f8eb31ca0364fcb60438 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Wed, 25 Jul 2018 23:07:13 -0700 Subject: [PATCH 408/453] version 0.15.1 --- multipart/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index bcc057259..3b42e8af2 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "multipart" -version = "0.15.0" +version = "0.15.1" authors = ["Austin Bonander "] From 513eaa2962e7fbabc73a10c5d3e0d783050c971d Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Thu, 26 Jul 2018 03:37:44 -0700 Subject: [PATCH 409/453] copy repro of abonander/multipart#104 --- multipart/src/server/mod.rs | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index fb4cfb8b7..bf3f5e41e 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -212,3 +212,30 @@ pub trait HttpRequest { /// Return the request body for reading. fn body(self) -> Self::Body; } + +#[test] +fn issue_104() { + let logger = ::mock::log_on_panic(); + + use std::io::Cursor; + + let body = "\ + POST /test.html HTTP/1.1\r\n\ + Host: example.org\r\n\ + Content-Type: multipart/form-data;boundary=\"boundary\"\r\n\r\n\ + Content-Disposition: form-data; name=\"field1\"\r\n\r\n\ + value1\r\n\ + Content-Disposition: form-data; name=\"field2\"; filename=\"example.txt\"\r\n\r\n\ + value2 "; + + let request = Cursor::new(body); + + let mut multipart = Multipart::with_body(request, "boundary"); + let multipart_result = multipart.foreach_entry(|_field| { + // Do nothing + }); + + println!("{:?}", multipart_result); + + logger.clear(); +} From 045dadb29223e9abb6435254307814e4954ec105 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Thu, 26 Jul 2018 04:03:00 -0700 Subject: [PATCH 410/453] drop `PanicLogger`, revert to `env_logger` --- multipart/src/lib.rs | 8 ++++ multipart/src/local_test.rs | 4 +- multipart/src/mock.rs | 72 -------------------------------- multipart/src/server/boundary.rs | 22 +++------- multipart/src/server/mod.rs | 4 +- 5 files changed, 15 insertions(+), 95 deletions(-) diff --git a/multipart/src/lib.rs b/multipart/src/lib.rs index 8f4eca6d8..0b04711b7 100644 --- a/multipart/src/lib.rs +++ b/multipart/src/lib.rs @@ -67,6 +67,9 @@ extern crate iron; #[cfg(feature = "tiny_http")] extern crate tiny_http; +#[cfg(test)] +extern crate env_logger; + #[cfg(any(feature = "mock", test))] pub mod mock; @@ -118,3 +121,8 @@ mod local_test; fn random_alphanumeric(len: usize) -> String { rand::thread_rng().gen_ascii_chars().take(len).collect() } + +#[cfg(test)] +fn init_log() { + let _ = env_logger::try_init(); +} \ No newline at end of file diff --git a/multipart/src/local_test.rs b/multipart/src/local_test.rs index 5dbf4bf20..935a0462f 100644 --- a/multipart/src/local_test.rs +++ b/multipart/src/local_test.rs @@ -205,7 +205,7 @@ impl fmt::Debug for PrintHex { macro_rules! do_test ( ($client_test:ident, $server_test:ident) => ( - let panic_logger = ::mock::log_on_panic(); + ::init_log(); info!("Client Test: {:?} Server Test: {:?}", stringify!($client_test), stringify!($server_test)); @@ -224,8 +224,6 @@ macro_rules! do_test ( $server_test(buf, &mut test_fields); test_fields.assert_is_empty(); - - panic_logger.clear(); ); ); diff --git a/multipart/src/mock.rs b/multipart/src/mock.rs index 57dad3cfe..b75467b18 100644 --- a/multipart/src/mock.rs +++ b/multipart/src/mock.rs @@ -203,75 +203,3 @@ impl<'s, W: Write> Write for StdoutTee<'s, W> { self.stdout.flush() } } - -/// Capture all logging on this thread during `PanicLogger`'s lifetime. -/// -/// When `PanicLogger` is dropped: if the thread is panicking, all captured logs are printed -/// to stdout, otherwise the captured logs are deleted. -/// -/// RFC: break this functionality out into its own crate? Could also support nested invocations -/// (where each instance of `PanicLogger` only prints the logs captured during its lifetime) -/// but that's currently unused here. -pub fn log_on_panic() -> PanicLogger { - ::log::set_max_level(::log::LevelFilter::Trace); - - if ::log::logger() as *const ::log::Log != &LOGGER as *const ::log::Log { - ::log::set_logger(&LOGGER).expect("failed to set logger for `log_on_panic()`"); - } - - LOG_ENABLED.with(|flag| flag.set(true)); - - PanicLogger(()) -} - -/// Struct that logs if the thread panics before it is dropped. -#[must_use] -pub struct PanicLogger(()); - -impl PanicLogger { - /// Clear the buffered logs - pub fn clear(self) { - drop(self); - } -} - -impl Drop for PanicLogger { - fn drop(&mut self) { - LOG_ENABLED.with(|flag| flag.set(false)); - LOG.with(|log| { - let mut log = log.borrow_mut(); - if thread::panicking() { - println!("captured logs before panic:\n{}", *log); - } - log.clear(); - }); - } -} - -thread_local! { - static LOG: RefCell = RefCell::new(String::new()); - static LOG_ENABLED: Cell = Cell::new(false); -} - -static LOGGER: ThreadLocalLogger = ThreadLocalLogger; - -struct ThreadLocalLogger; - -impl ::log::Log for ThreadLocalLogger { - fn enabled(&self, _: &Metadata) -> bool { - LOG_ENABLED.with(Cell::get) - } - - fn log(&self, record: &Record) { - if self.enabled(record.metadata()) { - LOG.with(|log| { - // format the message before pushing it to `ptr` since the fmt - // impls of the record arguments could try to log as well - let msg = format!("{}:{}: {}\n", record.level(), record.target(), record.args()); - log.borrow_mut().push_str(&msg); - }); - } - } - - fn flush(&self) {} -} diff --git a/multipart/src/server/boundary.rs b/multipart/src/server/boundary.rs index 342eb4d47..8205f8c0a 100644 --- a/multipart/src/server/boundary.rs +++ b/multipart/src/server/boundary.rs @@ -240,7 +240,7 @@ mod test { #[test] fn test_boundary() { - let _logger = ::mock::log_on_panic(); + ::init_log(); debug!("Testing boundary (no split)"); @@ -286,7 +286,7 @@ mod test { #[test] fn test_split_boundary() { - let logger = ::mock::log_on_panic(); + ::init_log(); debug!("Testing boundary (split)"); @@ -300,8 +300,6 @@ mod test { let mut reader = BoundaryReader::from_reader(src, BOUNDARY); test_boundary_reader(&mut reader, &mut buf); } - - logger.clear(); } fn test_boundary_reader(reader: &mut BoundaryReader, buf: &mut String) { @@ -338,8 +336,6 @@ mod test { #[test] fn test_leading_crlf() { - let logger = ::mock::log_on_panic(); - let mut body: &[u8] = b"\r\n\r\n--boundary\r\n\ asdf1234\ \r\n\r\n--boundary--"; @@ -362,13 +358,11 @@ mod test { debug!("Read 2 (empty)"); let _ = reader.read_to_string(buf).unwrap(); assert_eq!(buf, ""); - - logger.clear() } #[test] fn test_trailing_crlf() { - let logger = ::mock::log_on_panic(); + ::init_log(); let mut body: &[u8] = b"--boundary\r\n\ asdf1234\ @@ -407,14 +401,12 @@ mod test { debug!("Read 3 (empty)"); let _ = reader.read_to_string(buf).unwrap(); assert_eq!(buf, ""); - - logger.clear(); } // https://github.com/abonander/multipart/issues/93#issuecomment-343610587 #[test] fn test_trailing_lflf() { - let logger = ::mock::log_on_panic(); + ::init_log(); let mut body: &[u8] = b"--boundary\r\n\ asdf1234\ @@ -452,14 +444,12 @@ mod test { debug!("Read 3 (empty)"); let _ = reader.read_to_string(buf).unwrap(); assert_eq!(buf, ""); - - logger.clear(); } // https://github.com/abonander/multipart/issues/104 #[test] fn test_unterminated_body() { - let logger = ::mock::log_on_panic(); + ::init_log(); let mut body: &[u8] = b"--boundary\r\n\ asdf1234\ @@ -488,8 +478,6 @@ mod test { debug!("Read 2"); let _ = reader.read_to_string(buf).unwrap_err(); - - logger.clear(); } #[cfg(feature = "bench")] diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index bf3f5e41e..ab526ced6 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -215,7 +215,7 @@ pub trait HttpRequest { #[test] fn issue_104() { - let logger = ::mock::log_on_panic(); + ::init_log(); use std::io::Cursor; @@ -236,6 +236,4 @@ fn issue_104() { }); println!("{:?}", multipart_result); - - logger.clear(); } From d5380786919bbfdfd8c6fade674cb0966dfff17e Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sat, 28 Jul 2018 03:40:06 -0700 Subject: [PATCH 411/453] fix abonander/multipart#104 in server/boundary.rs return errors if the buffer is too small, return as much data as possible between reads --- multipart/src/mock.rs | 2 - multipart/src/server/boundary.rs | 94 ++++++++++++++++++++------------ multipart/src/server/mod.rs | 8 +-- 3 files changed, 61 insertions(+), 43 deletions(-) diff --git a/multipart/src/mock.rs b/multipart/src/mock.rs index b75467b18..f47dedeba 100644 --- a/multipart/src/mock.rs +++ b/multipart/src/mock.rs @@ -9,8 +9,6 @@ use std::cell::{Cell, RefCell}; use std::io::{self, Read, Write}; use std::{fmt, thread}; -use log::{Metadata, Record}; - use rand::{self, Rng, ThreadRng}; /// A mock implementation of `client::HttpRequest` which can spawn an `HttpBuffer`. diff --git a/multipart/src/server/boundary.rs b/multipart/src/server/boundary.rs index 8205f8c0a..0095cb2e6 100644 --- a/multipart/src/server/boundary.rs +++ b/multipart/src/server/boundary.rs @@ -55,57 +55,49 @@ impl BoundaryReader where R: Read { } fn read_to_boundary(&mut self) -> io::Result<&[u8]> { - if self.state == AtEnd { return Ok(&[])} - let buf = self.source.fill_buf()?; - if self.search_idx == 0 && buf.len() < self.boundary.len() { - return Err(io::Error::new(io::ErrorKind::UnexpectedEof, - "unexpected end of request body")); - } - trace!("Buf: {:?}", String::from_utf8_lossy(buf)); - debug!("Before-loop Buf len: {} Search idx: {} State: {:?}", + debug!("Before search Buf len: {} Search idx: {} State: {:?}", buf.len(), self.search_idx, self.state); + if self.state == BoundaryRead || self.state == AtEnd { + return Ok(&buf[..self.search_idx]) + } + if self.state == Searching && self.search_idx < buf.len() { let lookahead = &buf[self.search_idx..]; - debug!("Find boundary loop! Lookahead len: {}", lookahead.len()); - // Look for the boundary, or if it isn't found, stop near the end. - match twoway::find_bytes(lookahead, &self.boundary) { - Some(found_idx) => { + match find_boundary(lookahead, &self.boundary) { + Ok(found_idx) => { self.search_idx += found_idx; self.state = BoundaryRead; }, - None => { - self.search_idx += lookahead.len().saturating_sub(self.boundary.len() + 2); + Err(yield_len) => { + self.search_idx += yield_len; } } } - debug!("After-loop Buf len: {} Search idx: {} State: {:?}", + debug!("After search Buf len: {} Search idx: {} State: {:?}", buf.len(), self.search_idx, self.state); - // don't modify search_idx so it always points to the start of the boundary - let mut buf_len = self.search_idx; - - // back up the cursor to before the boundary's preceding CRLF - if self.state != Searching && buf_len >= 2 { - let two_bytes_before = &buf[buf_len - 2 .. buf_len]; + // back up the cursor to before the boundary's preceding CRLF if we haven't already + if self.search_idx >= 2 && !buf[self.search_idx..].starts_with(b"\r\n") { + let two_bytes_before = &buf[self.search_idx - 2 .. self.search_idx]; trace!("Two bytes before: {:?} ({:?}) (\"\\r\\n\": {:?})", String::from_utf8_lossy(two_bytes_before), two_bytes_before, b"\r\n"); if two_bytes_before == *b"\r\n" { debug!("Subtract two!"); - buf_len -= 2; + self.search_idx -= 2; } } - let ret_buf = &buf[..buf_len]; + let ret_buf = &buf[..self.search_idx]; trace!("Returning buf: {:?}", String::from_utf8_lossy(ret_buf)); @@ -130,17 +122,24 @@ impl BoundaryReader where R: Read { let buf_len = self.read_to_boundary()?.len(); + if buf_len == 0 && self.state == Searching { + return Err(io::Error::new(io::ErrorKind::UnexpectedEof, + "unexpected end of request body")); + } + debug!("Discarding {} bytes", buf_len); self.consume(buf_len); } let consume_amt = { - let min_len = self.boundary.len() + 4; - let buf = self.source.fill_buf()?; - if buf.len() < min_len { + // consume everything up to and including the boundary leading CR-LF and + // trailing two bytes + let mut consume_amt = self.search_idx + self.boundary.len() + 4; + + if buf.len() < consume_amt { return Err(io::Error::new(io::ErrorKind::UnexpectedEof, "not enough bytes to verify boundary")); } @@ -148,15 +147,17 @@ impl BoundaryReader where R: Read { // we have enough bytes to verify self.state = Searching; - let mut consume_amt = self.search_idx + self.boundary.len(); - - let last_two = &buf[consume_amt .. consume_amt + 2]; + let last_two = &buf[consume_amt - 2 .. consume_amt]; match last_two { - b"\r\n" => consume_amt += 2, - b"--" => { consume_amt += 2; self.state = AtEnd }, - _ => debug!("Unexpected bytes following boundary: {:?}", - String::from_utf8_lossy(&last_two)), + b"\r\n" => (), + b"--" => self.state = AtEnd, + _ => { + // don't consume unexpected bytes + consume_amt -= 2; + warn!("Unexpected bytes following boundary: {:X} {:X}", + last_two[0], last_two[1]); + }, } consume_amt @@ -176,6 +177,24 @@ impl BoundaryReader where R: Read { } } +/// Find the boundary occurrence or the highest length to safely yield +fn find_boundary(buf: &[u8], boundary: &[u8]) -> Result { + if let Some(idx) = twoway::find_bytes(buf, boundary) { + return Ok(idx); + } + + let search_start = buf.len().saturating_sub(boundary.len()); + + // search for just the boundary fragment + for i in search_start .. buf.len() { + if boundary.starts_with(&buf[i..]) { + return Err(i); + } + } + + Err(buf.len()) +} + #[cfg(feature = "bench")] impl<'a> BoundaryReader> { fn new_with_bytes(bytes: &'a [u8], boundary: &str) -> Self { @@ -454,7 +473,7 @@ mod test { let mut body: &[u8] = b"--boundary\r\n\ asdf1234\ \n\n\r\n--boundary\r\n\ - hjkl5678"; + hjkl5678 "; let ref mut buf = String::new(); let mut reader = BoundaryReader::from_reader(&mut body, BOUNDARY); @@ -477,7 +496,12 @@ mod test { reader.consume_boundary().unwrap(); debug!("Read 2"); - let _ = reader.read_to_string(buf).unwrap_err(); + let _ = reader.read_to_string(buf).unwrap(); + assert_eq!(buf, "hjkl5678 "); + buf.clear(); + + debug!("Consume 3 - expecting error"); + reader.consume_boundary().unwrap_err(); } #[cfg(feature = "bench")] diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index ab526ced6..d9db94cd0 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -123,7 +123,7 @@ impl Multipart { pub fn with_body>(body: R, boundary: Bnd) -> Self { let boundary = boundary.into(); - info!("Multipart::with_boundary(_, {:?}", boundary); + info!("Multipart::with_boundary(_, {:?})", boundary); Multipart { reader: BoundaryReader::from_reader(body, boundary), @@ -231,9 +231,5 @@ fn issue_104() { let request = Cursor::new(body); let mut multipart = Multipart::with_body(request, "boundary"); - let multipart_result = multipart.foreach_entry(|_field| { - // Do nothing - }); - - println!("{:?}", multipart_result); + multipart.foreach_entry(|_field| {/* Do nothing */}).unwrap_err(); } From 581c250a2681610c12bf50a5aec4d397cd60a6ad Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sun, 29 Jul 2018 17:48:04 -0700 Subject: [PATCH 412/453] 0.15.2: merged abonander/multipart#105 --- multipart/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 3b42e8af2..e78f7ffba 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "multipart" -version = "0.15.1" +version = "0.15.2" authors = ["Austin Bonander "] From 6d8c9a6d770d2cc1a87954e28d7e9b20ba3ed61c Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Fri, 24 Aug 2018 04:29:45 -0700 Subject: [PATCH 413/453] (failing) implement test form, test empty body with only closing boundary --- multipart/Cargo.toml | 4 +++ multipart/src/bin/form_test.rs | 30 +++++++++++++++++++ .../bin/{test_multipart.rs => read_file.rs} | 0 multipart/src/bin/test_form.html | 14 +++++++++ multipart/src/server/boundary.rs | 21 +++++++++++++ 5 files changed, 69 insertions(+) create mode 100644 multipart/src/bin/form_test.rs rename multipart/src/bin/{test_multipart.rs => read_file.rs} (100%) create mode 100644 multipart/src/bin/test_form.html diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index e78f7ffba..7aaca74bd 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -89,3 +89,7 @@ required-features = ["mock", "tiny_http", "server"] [[example]] name = "rocket" required-features = ["mock", "rocket", "rocket_codegen", "server"] + +[[bin]] +name = "form_test" +required-features = ["mock", "hyper", "server"] \ No newline at end of file diff --git a/multipart/src/bin/form_test.rs b/multipart/src/bin/form_test.rs new file mode 100644 index 000000000..0f0241f35 --- /dev/null +++ b/multipart/src/bin/form_test.rs @@ -0,0 +1,30 @@ +extern crate hyper; +extern crate multipart; + +use multipart::server::Multipart; + +use hyper::header::ContentType; +use hyper::server::*; + +use std::fs::File; +use std::io; + +fn main() { + let _ = Server::http("127.0.0.1:8080").expect("failed to bind socket") + .handle(read_multipart).expect("failed to handle request"); +} + +fn read_multipart(req: Request, mut resp: Response) { + if let Ok(mut multipart) = Multipart::from_request(req) { + multipart.foreach_entry(|_| {}) + .map_err(|e| println!("error handling field: {}", e)); + } + + let mut file = File::open("src/bin/test_form.html") + .expect("failed to open src/bind/test_form.html"); + + resp.headers_mut().set(ContentType("text/html".parse().unwrap())); + + let mut resp = resp.start().expect("failed to open response"); + io::copy(&mut file, &mut resp).expect("failed to write response"); +} \ No newline at end of file diff --git a/multipart/src/bin/test_multipart.rs b/multipart/src/bin/read_file.rs similarity index 100% rename from multipart/src/bin/test_multipart.rs rename to multipart/src/bin/read_file.rs diff --git a/multipart/src/bin/test_form.html b/multipart/src/bin/test_form.html new file mode 100644 index 000000000..6d10065b7 --- /dev/null +++ b/multipart/src/bin/test_form.html @@ -0,0 +1,14 @@ + + + + + Multipart-Async Form Test + + +
+ + +
+ + diff --git a/multipart/src/server/boundary.rs b/multipart/src/server/boundary.rs index 0095cb2e6..429004e5a 100644 --- a/multipart/src/server/boundary.rs +++ b/multipart/src/server/boundary.rs @@ -353,6 +353,27 @@ mod test { assert_eq!(buf, ""); } + #[test] + fn test_empty_body() { + // empty body contains closing boundary only + let mut body: &[u8] = b"--boundary--"; + + let ref mut buf = String::new(); + let mut reader = BoundaryReader::from_reader(&mut body, BOUNDARY); + + debug!("Consume 1"); + let ret = reader.consume_boundary().unwrap(); + assert_eq!(ret, false); + + debug!("Read 1"); + let _ = reader.read_to_string(buf).unwrap(); + assert_eq!(buf, ""); + buf.clear(); + + debug!("Consume 2"); + assert!(reader.consume_boundary().is_err()); + } + #[test] fn test_leading_crlf() { let mut body: &[u8] = b"\r\n\r\n--boundary\r\n\ From c8cf94949377dc14c9e88198323df2e3b02e6a9f Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Mon, 27 Aug 2018 19:47:37 -0700 Subject: [PATCH 414/453] fix empty body test (inverted the meaning of `PrivReadEntry::consume_boundary()`) --- multipart/src/server/boundary.rs | 44 ++++++++++++++++++-------------- multipart/src/server/field.rs | 4 +-- multipart/src/server/mod.rs | 2 +- 3 files changed, 28 insertions(+), 22 deletions(-) diff --git a/multipart/src/server/boundary.rs b/multipart/src/server/boundary.rs index 429004e5a..7f1fe51b6 100644 --- a/multipart/src/server/boundary.rs +++ b/multipart/src/server/boundary.rs @@ -111,10 +111,9 @@ impl BoundaryReader where R: Read { self.source.policy_mut().0 = min_buf_size; } - #[doc(hidden)] pub fn consume_boundary(&mut self) -> io::Result { if self.state == AtEnd { - return Ok(true); + return Ok(false); } while self.state == Searching { @@ -135,9 +134,13 @@ impl BoundaryReader where R: Read { let consume_amt = { let buf = self.source.fill_buf()?; - // consume everything up to and including the boundary leading CR-LF and - // trailing two bytes - let mut consume_amt = self.search_idx + self.boundary.len() + 4; + // consume everything up to and including the boundary and trailing CRLF/`--` + let mut consume_amt = self.search_idx + self.boundary.len() + 2; + + // if our cursor stopped on the preceding CRLF, include it in the consume + if buf.starts_with(b"\r\n") { + consume_amt += 2; + } if buf.len() < consume_amt { return Err(io::Error::new(io::ErrorKind::UnexpectedEof, @@ -173,7 +176,7 @@ impl BoundaryReader where R: Read { trace!("Consumed boundary (state: {:?}), remaining buf: {:?}", self.state, String::from_utf8_lossy(self.source.buffer())); - Ok(self.state == AtEnd) + Ok(self.state != AtEnd) } } @@ -355,6 +358,8 @@ mod test { #[test] fn test_empty_body() { + ::init_log(); + // empty body contains closing boundary only let mut body: &[u8] = b"--boundary--"; @@ -362,8 +367,7 @@ mod test { let mut reader = BoundaryReader::from_reader(&mut body, BOUNDARY); debug!("Consume 1"); - let ret = reader.consume_boundary().unwrap(); - assert_eq!(ret, false); + assert_eq!(reader.consume_boundary().unwrap(), false); debug!("Read 1"); let _ = reader.read_to_string(buf).unwrap(); @@ -371,11 +375,13 @@ mod test { buf.clear(); debug!("Consume 2"); - assert!(reader.consume_boundary().is_err()); + assert_eq!(reader.consume_boundary().unwrap(), false); } #[test] fn test_leading_crlf() { + ::init_log(); + let mut body: &[u8] = b"\r\n\r\n--boundary\r\n\ asdf1234\ \r\n\r\n--boundary--"; @@ -385,7 +391,7 @@ mod test { debug!("Consume 1"); - reader.consume_boundary().unwrap(); + assert_eq!(reader.consume_boundary().unwrap(), true); debug!("Read 1"); let _ = reader.read_to_string(buf).unwrap(); @@ -393,7 +399,7 @@ mod test { buf.clear(); debug!("Consume 2"); - reader.consume_boundary().unwrap(); + assert_eq!(reader.consume_boundary().unwrap(), false); debug!("Read 2 (empty)"); let _ = reader.read_to_string(buf).unwrap(); @@ -413,7 +419,7 @@ mod test { let mut reader = BoundaryReader::from_reader(&mut body, BOUNDARY); debug!("Consume 1"); - reader.consume_boundary().unwrap(); + assert_eq!(reader.consume_boundary().unwrap(), true); debug!("Read 1"); @@ -428,7 +434,7 @@ mod test { buf.clear(); debug!("Consume 2"); - reader.consume_boundary().unwrap(); + assert_eq!(reader.consume_boundary().unwrap(), true); debug!("Read 2"); let _ = reader.read_to_string(buf).unwrap(); @@ -436,7 +442,7 @@ mod test { buf.clear(); debug!("Consume 3"); - reader.consume_boundary().unwrap(); + assert_eq!(reader.consume_boundary().unwrap(), false); debug!("Read 3 (empty)"); let _ = reader.read_to_string(buf).unwrap(); @@ -457,7 +463,7 @@ mod test { let mut reader = BoundaryReader::from_reader(&mut body, BOUNDARY); debug!("Consume 1"); - reader.consume_boundary().unwrap(); + assert_eq!(reader.consume_boundary().unwrap(), true); debug!("Read 1"); @@ -471,7 +477,7 @@ mod test { buf.clear(); debug!("Consume 2"); - reader.consume_boundary().unwrap(); + assert_eq!(reader.consume_boundary().unwrap(), true); debug!("Read 2"); let _ = reader.read_to_string(buf).unwrap(); @@ -479,7 +485,7 @@ mod test { buf.clear(); debug!("Consume 3"); - reader.consume_boundary().unwrap(); + assert_eq!(reader.consume_boundary().unwrap(), false); debug!("Read 3 (empty)"); let _ = reader.read_to_string(buf).unwrap(); @@ -500,7 +506,7 @@ mod test { let mut reader = BoundaryReader::from_reader(&mut body, BOUNDARY); debug!("Consume 1"); - reader.consume_boundary().unwrap(); + assert_eq!(reader.consume_boundary().unwrap(), true); debug!("Read 1"); @@ -514,7 +520,7 @@ mod test { buf.clear(); debug!("Consume 2"); - reader.consume_boundary().unwrap(); + assert_eq!(reader.consume_boundary().unwrap(), true); debug!("Read 2"); let _ = reader.read_to_string(buf).unwrap(); diff --git a/multipart/src/server/field.rs b/multipart/src/server/field.rs index 0c25c638c..32222c743 100644 --- a/multipart/src/server/field.rs +++ b/multipart/src/server/field.rs @@ -364,7 +364,7 @@ pub trait ReadEntry: PrivReadEntry + Sized { debug!("ReadEntry::read_entry()"); - if try_read_entry!(self; self.consume_boundary()) { + if !try_read_entry!(self; self.consume_boundary()) { return End(self); } @@ -409,7 +409,7 @@ pub trait PrivReadEntry { fn set_min_buf_size(&mut self, min_buf_size: usize); /// Consume the next boundary. - /// Returns `true` if the last boundary was read, `false` otherwise. + /// Returns `true` if a field should follow, `false` otherwise. fn consume_boundary(&mut self) -> io::Result; fn read_headers(&mut self) -> Result { diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index d9db94cd0..0ce725e38 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -188,7 +188,7 @@ impl PrivReadEntry for Multipart { } /// Consume the next boundary. - /// Returns `true` if the last boundary was read, `false` otherwise. + /// Returns `true` if a field should follow this boundary, `false` otherwise. fn consume_boundary(&mut self) -> io::Result { debug!("Consume boundary!"); self.reader.consume_boundary() From 2ac532852a824a1d8250da8db2de0f920634e50d Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Mon, 27 Aug 2018 19:48:08 -0700 Subject: [PATCH 415/453] form_test: don't bind to a static port --- multipart/src/bin/form_test.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/multipart/src/bin/form_test.rs b/multipart/src/bin/form_test.rs index 0f0241f35..348797642 100644 --- a/multipart/src/bin/form_test.rs +++ b/multipart/src/bin/form_test.rs @@ -10,8 +10,10 @@ use std::fs::File; use std::io; fn main() { - let _ = Server::http("127.0.0.1:8080").expect("failed to bind socket") + let listening = Server::http("127.0.0.1:0").expect("failed to bind socket") .handle(read_multipart).expect("failed to handle request"); + + println!("bound socket to: {}", listening.socket); } fn read_multipart(req: Request, mut resp: Response) { @@ -27,4 +29,4 @@ fn read_multipart(req: Request, mut resp: Response) { let mut resp = resp.start().expect("failed to open response"); io::copy(&mut file, &mut resp).expect("failed to write response"); -} \ No newline at end of file +} From 9a319100f5a9a6e59f837371c7d04e1d69bbfcfe Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Mon, 27 Aug 2018 19:51:01 -0700 Subject: [PATCH 416/453] client: always write closing boundary (matches Chrome's behavior) --- multipart/src/client/mod.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/multipart/src/client/mod.rs b/multipart/src/client/mod.rs index 40d535130..325f21bde 100644 --- a/multipart/src/client/mod.rs +++ b/multipart/src/client/mod.rs @@ -253,10 +253,11 @@ impl<'a, W: Write> MultipartWriter<'a, W> { fn finish(mut self) -> io::Result { if self.data_written { - // Write two hyphens after the last boundary occurrence. - write!(self.inner, "\r\n--{}--", self.boundary)?; + self.inner.write_all(b"\r\n")?; } + // always write the closing boundary, even for empty bodies + write!(self.inner, "--{}--", self.boundary)?; Ok(self.inner) } } From 1a6b5c2469b2bbfdd2a03e4f043fc15b4694f6ab Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Mon, 27 Aug 2018 19:51:51 -0700 Subject: [PATCH 417/453] 0.15.3: client and server, fix handling of empty request bodies (should always contain end boundary) --- multipart/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 7aaca74bd..516b4c1d8 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "multipart" -version = "0.15.2" +version = "0.15.3" authors = ["Austin Bonander "] @@ -92,4 +92,4 @@ required-features = ["mock", "rocket", "rocket_codegen", "server"] [[bin]] name = "form_test" -required-features = ["mock", "hyper", "server"] \ No newline at end of file +required-features = ["mock", "hyper", "server"] From 714526b3d5b84fdb066ad3ddcacde8a414b6f21e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jess=C3=A9=20Vermeulen?= <43927190+JesseVermeulen123@users.noreply.github.com> Date: Mon, 15 Oct 2018 20:53:16 +0200 Subject: [PATCH 418/453] Fix GitHub-Flavoured Markdown --- multipart/examples/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/multipart/examples/README.md b/multipart/examples/README.md index 5f0b6999b..ebad14068 100644 --- a/multipart/examples/README.md +++ b/multipart/examples/README.md @@ -5,7 +5,7 @@ These example files show how to use `multipart` with the various crates it integ These files carry the same licenses as [`multipart` itself](https://github.com/abonander/multipart#license), though this may be lightened to a copyright-free license in the near future. -##Client +## Client Examples for the client-side integrations of `multipart`'s API. @@ -30,7 +30,7 @@ via the lazy-writing capabilities of `multipart::client::lazy`. $ cargo run --example hyper_reqbuilder ``` -##Server +## Server [`hyper_server`](hyper_server.rs) --------------------------------- From 72adba449cea624d08d8b6984490cc06013a1e90 Mon Sep 17 00:00:00 2001 From: Michael Zhang Date: Tue, 20 Nov 2018 02:48:46 -0600 Subject: [PATCH 419/453] tack on final \r\n --- multipart/src/client/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/src/client/mod.rs b/multipart/src/client/mod.rs index 325f21bde..6446be29a 100644 --- a/multipart/src/client/mod.rs +++ b/multipart/src/client/mod.rs @@ -257,7 +257,7 @@ impl<'a, W: Write> MultipartWriter<'a, W> { } // always write the closing boundary, even for empty bodies - write!(self.inner, "--{}--", self.boundary)?; + write!(self.inner, "--{}--\r\n", self.boundary)?; Ok(self.inner) } } From 99bc86c8bd7c667e198734bedf5b24ffaddef02b Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Tue, 27 Nov 2018 01:47:41 -0800 Subject: [PATCH 420/453] add repro for #114, some debug asserts to narrow it down --- multipart/src/server/boundary.rs | 16 ++++++- multipart/src/server/mod.rs | 72 ++++++++++++++++++++++++++++++++ 2 files changed, 87 insertions(+), 1 deletion(-) diff --git a/multipart/src/server/boundary.rs b/multipart/src/server/boundary.rs index 7f1fe51b6..3609473d6 100644 --- a/multipart/src/server/boundary.rs +++ b/multipart/src/server/boundary.rs @@ -40,7 +40,7 @@ pub struct BoundaryReader { } impl BoundaryReader where R: Read { - #[doc(hidden)] + /// Internal API pub fn from_reader>>(reader: R, boundary: B) -> BoundaryReader { let mut boundary = boundary.into(); safemem::prepend(b"--", &mut boundary); @@ -140,6 +140,10 @@ impl BoundaryReader where R: Read { // if our cursor stopped on the preceding CRLF, include it in the consume if buf.starts_with(b"\r\n") { consume_amt += 2; + + debug_assert_eq!(*self.boundary, buf[2..2 + self.boundary.len()]); + } else { + debug_assert_eq!(*self.boundary, buf[..self.boundary.len()]); } if buf.len() < consume_amt { @@ -156,6 +160,11 @@ impl BoundaryReader where R: Read { b"\r\n" => (), b"--" => self.state = AtEnd, _ => { + if cfg!(debug_assertions) { + panic!("Unexpected bytes following boundary: {:X} {:X}", + last_two[0], last_two[1]); + } + // don't consume unexpected bytes consume_amt -= 2; warn!("Unexpected bytes following boundary: {:X} {:X}", @@ -171,6 +180,11 @@ impl BoundaryReader where R: Read { String::from_utf8_lossy(self.source.buffer())); self.source.consume(consume_amt); + + if cfg!(debug_assertions) { + + } + self.search_idx = 0; trace!("Consumed boundary (state: {:?}), remaining buf: {:?}", self.state, diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index 0ce725e38..c9900e365 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -233,3 +233,75 @@ fn issue_104() { let mut multipart = Multipart::with_body(request, "boundary"); multipart.foreach_entry(|_field| {/* Do nothing */}).unwrap_err(); } + +#[test] +fn issue_114() { + ::init_log(); + + fn consume_all(mut rdr: R) { + let mut consume = 0; + + loop { + let consume = rdr.fill_buf().unwrap().len(); + if consume == 0 { return; } + rdr.consume(consume); + } + } + + use std::io::Cursor; + + let body = "\ + --------------------------c616e5fded96a3c7\r\n\ + Content-Disposition: form-data; name=\"key1\"\r\n\r\n\ + v1,\r\n\ + --------------------------c616e5fded96a3c7\r\n\ + Content-Disposition: form-data; name=\"key2\"\r\n\r\n\ + v2,\r\n\ + --------------------------c616e5fded96a3c7\r\n\ + Content-Disposition: form-data; name=\"key3\"\r\n\r\n\ + v3\r\n\ + --------------------------c616e5fded96a3c7--\r\n"; + + let request = Cursor::new(body); + let mut multipart = Multipart::with_body(request, "------------------------c616e5fded96a3c7"); + + // one error if you do nothing + multipart.foreach_entry(|_entry| { /* do nothing */}).unwrap(); + + // a different error if you skip the first field + multipart.foreach_entry(|mut entry| if *entry.headers.name != "key1" { consume_all(entry.data); }) + .unwrap(); + + + multipart.foreach_entry(|mut entry| () /* match entry.headers.name.as_str() { + "file" => { + let mut vec = Vec::new(); + entry.data.read_to_end(&mut vec).expect("can't read"); + // message.file = String::from_utf8(vec).ok(); + println!("key file got"); + } + + "key1" => { + let mut vec = Vec::new(); + entry.data.read_to_end(&mut vec).expect("can't read"); + // message.key1 = String::from_utf8(vec).ok(); + println!("key1 got"); + } + + "key2" => { + let mut vec = Vec::new(); + entry.data.read_to_end(&mut vec).expect("can't read"); + // message.key2 = String::from_utf8(vec).ok(); + println!("key2 got"); + } + + _ => { + // as multipart has a bug https://github.com/abonander/multipart/issues/114 + // we manually do read_to_end here + //let mut _vec = Vec::new(); + //entry.data.read_to_end(&mut _vec).expect("can't read"); + println!("key neglected"); + } + }*/) + .expect("Unable to iterate multipart?") +} From 4b798a5e8a9126cceacbe9715c3d0793d4292532 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Wed, 28 Nov 2018 03:23:54 -0800 Subject: [PATCH 421/453] add fixes and regression tests for abonander/multipart#114 --- multipart/src/server/boundary.rs | 68 +++++++++++++++++++++++--------- 1 file changed, 50 insertions(+), 18 deletions(-) diff --git a/multipart/src/server/boundary.rs b/multipart/src/server/boundary.rs index 3609473d6..21f37d14d 100644 --- a/multipart/src/server/boundary.rs +++ b/multipart/src/server/boundary.rs @@ -134,18 +134,26 @@ impl BoundaryReader where R: Read { let consume_amt = { let buf = self.source.fill_buf()?; - // consume everything up to and including the boundary and trailing CRLF/`--` - let mut consume_amt = self.search_idx + self.boundary.len() + 2; + // if the boundary is found we should have at least this much in-buffer + let mut consume_amt = self.search_idx + self.boundary.len(); - // if our cursor stopped on the preceding CRLF, include it in the consume - if buf.starts_with(b"\r\n") { + // we don't care about data before the cursor + let bnd_segment = &buf[self.search_idx..]; + + if bnd_segment.starts_with(b"\r\n") { + // preceding CRLF needs to be consumed as well consume_amt += 2; - debug_assert_eq!(*self.boundary, buf[2..2 + self.boundary.len()]); + // assert that we've found the boundary after the CRLF + debug_assert_eq!(*self.boundary, bnd_segment[2 .. self.boundary.len() + 2]); } else { - debug_assert_eq!(*self.boundary, buf[..self.boundary.len()]); + // assert that we've found the boundary + debug_assert_eq!(*self.boundary, bnd_segment[..self.boundary.len()]); } + // include the trailing CRLF or -- + consume_amt += 2; + if buf.len() < consume_amt { return Err(io::Error::new(io::ErrorKind::UnexpectedEof, "not enough bytes to verify boundary")); @@ -157,19 +165,13 @@ impl BoundaryReader where R: Read { let last_two = &buf[consume_amt - 2 .. consume_amt]; match last_two { - b"\r\n" => (), + b"\r\n" => self.state = Searching, b"--" => self.state = AtEnd, - _ => { - if cfg!(debug_assertions) { - panic!("Unexpected bytes following boundary: {:X} {:X}", - last_two[0], last_two[1]); - } - - // don't consume unexpected bytes - consume_amt -= 2; - warn!("Unexpected bytes following boundary: {:X} {:X}", - last_two[0], last_two[1]); - }, + _ => return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("unexpected bytes following multipart boundary: {:X} {:X}", + last_two[0], last_two[1]) + )), } consume_amt @@ -545,6 +547,36 @@ mod test { reader.consume_boundary().unwrap_err(); } + #[test] + fn test_lone_boundary() { + let mut body: &[u8] = b"--boundary"; + let mut reader = BoundaryReader::from_reader(&mut body, "boundary"); + reader.consume_boundary().unwrap_err(); + } + + #[test] + fn test_invalid_boundary() { + let mut body: &[u8] = b"--boundary\x00\x00"; + let mut reader = BoundaryReader::from_reader(&mut body, "boundary"); + reader.consume_boundary().unwrap_err(); + } + + #[test] + fn test_skip_field() { + let mut body: &[u8] = b"--boundary\r\nfield1\r\n--boundary\r\nfield2\r\n--boundary--"; + let mut reader = BoundaryReader::from_reader(&mut body, "boundary"); + + assert_eq!(reader.consume_boundary().unwrap(), true); + // skip `field1` + assert_eq!(reader.consume_boundary().unwrap(), true); + + let mut buf = String::new(); + reader.read_to_string(&mut buf).unwrap(); + assert_eq!(buf, "field2"); + + assert_eq!(reader.consume_boundary().unwrap(), false); + } + #[cfg(feature = "bench")] mod bench { extern crate test; From 052fab4af8c7269f724547ff511fddb6f0496064 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Thu, 29 Nov 2018 03:48:01 -0800 Subject: [PATCH 422/453] release patch version --- multipart/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 516b4c1d8..7dfd2449b 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "multipart" -version = "0.15.3" +version = "0.15.4" authors = ["Austin Bonander "] From a24c8c6305dedd9573a1620fa8d9c33fe5e31518 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Thu, 29 Nov 2018 03:50:20 -0800 Subject: [PATCH 423/453] bump minimum Rust version this already happened incidentally in `lazy_static 1.2.0` which was a semver-compatible upgrade --- multipart/.travis.yml | 2 +- multipart/README.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/multipart/.travis.yml b/multipart/.travis.yml index 1f6409adf..9bf245bf5 100644 --- a/multipart/.travis.yml +++ b/multipart/.travis.yml @@ -4,7 +4,7 @@ branches: except: - fuzzing rust: - - 1.22.1 + - 1.26.1 - stable - beta - nightly diff --git a/multipart/README.md b/multipart/README.md index b7243a3dd..88ad6c132 100644 --- a/multipart/README.md +++ b/multipart/README.md @@ -3,9 +3,9 @@ Client- and server-side abstractions for HTTP file uploads (POST requests with `Content-Type: multipart/form-data`). Supports several different (**sync**hronous API) HTTP crates. -**Async**hronous API support will be provided by [multipart-async]. +**Async**hronous (i.e. `futures`-based) API support will be provided by [multipart-async]. -Minimum supported Rust version: 1.22.1 +Minimum supported Rust version: 1.26.1 ### [Documentation](http://docs.rs/multipart/) From 1c6ca9ea2c8de41d928fb79e33d81e4a1db3d677 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Thu, 29 Nov 2018 15:40:09 -0800 Subject: [PATCH 424/453] try building on 1.22.1 --- multipart/.travis.yml | 9 +++++++++ multipart/README.md | 5 ++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/multipart/.travis.yml b/multipart/.travis.yml index 9bf245bf5..b30ca8da5 100644 --- a/multipart/.travis.yml +++ b/multipart/.travis.yml @@ -19,6 +19,9 @@ env: # ensure nickel works without hyper - ARGS+='--no-default-features --features "nickel"' - ARGS+='--features "use-arc-str"' +matrix: + include: + - rust: 1.22.1 script: - | if [ ${TRAVIS_RUST_VERSION} = "nightly" ]; then @@ -27,6 +30,12 @@ script: cargo test --verbose $ARGS --features "nightly,rocket,rocket_codegen" -- --test-threads=1; cargo doc --verbose $ARGS --features "nightly"; fi + - | + if [ ${TRAVIS_RUST_VERSION} = "1.22.1" ]; then + # we can't test because `env_logger` uses `lazy_static 1.2.0` which needs Rust 1.26 + cargo build --no-default-features --features "mock,client,server"; + exit 0; + fi - cargo build --verbose $ARGS; - cargo test --verbose $ARGS -- --test-threads=1; - cargo doc --verbose $ARGS; diff --git a/multipart/README.md b/multipart/README.md index 88ad6c132..aab85efc2 100644 --- a/multipart/README.md +++ b/multipart/README.md @@ -5,7 +5,10 @@ Client- and server-side abstractions for HTTP file uploads (POST requests with Supports several different (**sync**hronous API) HTTP crates. **Async**hronous (i.e. `futures`-based) API support will be provided by [multipart-async]. -Minimum supported Rust version: 1.26.1 +Minimum supported Rust version: 1.22.1* +* only `mock`, `client` and `server` features, only guaranteed to compile + +Fully tested Rust version: 1.26.1 ### [Documentation](http://docs.rs/multipart/) From 6da5d06a20548b3a4c66b548629d33bc1b7bf23a Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Thu, 29 Nov 2018 16:00:36 -0800 Subject: [PATCH 425/453] don't test these on every release --- multipart/.travis.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/multipart/.travis.yml b/multipart/.travis.yml index b30ca8da5..87176a49a 100644 --- a/multipart/.travis.yml +++ b/multipart/.travis.yml @@ -14,14 +14,13 @@ os: env: global: - RUST_LOG=multipart=trace RUST_BACKTRACE=1 ARGS= - matrix: - - ARGS+=--no-default-features - # ensure nickel works without hyper - - ARGS+='--no-default-features --features "nickel"' - - ARGS+='--features "use-arc-str"' matrix: include: - rust: 1.22.1 + - rust: stable + env: ARGS+=--no-default-features --features "nickel" + - rust: stable + env: ARGS+=--features "use_arc_str" script: - | if [ ${TRAVIS_RUST_VERSION} = "nightly" ]; then From c586875e16fee3cf65db7744cf512ba8211b4fe5 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Thu, 29 Nov 2018 16:02:17 -0800 Subject: [PATCH 426/453] simplify build script --- multipart/.travis.yml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/multipart/.travis.yml b/multipart/.travis.yml index 87176a49a..9cb4f92c9 100644 --- a/multipart/.travis.yml +++ b/multipart/.travis.yml @@ -21,14 +21,9 @@ matrix: env: ARGS+=--no-default-features --features "nickel" - rust: stable env: ARGS+=--features "use_arc_str" + - rust: nightly + env: ARGS+=--features "nightly,rocket,rocket_codegen" script: - - | - if [ ${TRAVIS_RUST_VERSION} = "nightly" ]; then - cargo build --verbose $ARGS --features "nightly"; - # test Rocket example as well - cargo test --verbose $ARGS --features "nightly,rocket,rocket_codegen" -- --test-threads=1; - cargo doc --verbose $ARGS --features "nightly"; - fi - | if [ ${TRAVIS_RUST_VERSION} = "1.22.1" ]; then # we can't test because `env_logger` uses `lazy_static 1.2.0` which needs Rust 1.26 From efee3803216b9deb07bc380b0b44e3fee01ad735 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Thu, 29 Nov 2018 16:39:54 -0800 Subject: [PATCH 427/453] turn off `slice-deque` feature of `buf_redux` --- multipart/.travis.yml | 3 +-- multipart/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/multipart/.travis.yml b/multipart/.travis.yml index 9cb4f92c9..9d7f2df96 100644 --- a/multipart/.travis.yml +++ b/multipart/.travis.yml @@ -27,8 +27,7 @@ script: - | if [ ${TRAVIS_RUST_VERSION} = "1.22.1" ]; then # we can't test because `env_logger` uses `lazy_static 1.2.0` which needs Rust 1.26 - cargo build --no-default-features --features "mock,client,server"; - exit 0; + cargo build --no-default-features --features "mock,client,server" && exit 0; fi - cargo build --verbose $ARGS; - cargo test --verbose $ARGS -- --test-threads=1; diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 7dfd2449b..2b89343ad 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -27,7 +27,7 @@ tempdir = "0.3" clippy = { version = ">=0.0, <0.1", optional = true} #Server Dependencies -buf_redux = { version = "0.7", optional = true } +buf_redux = { version = "0.7", optional = true, default-features = false } httparse = { version = "1.2", optional = true } twoway = { version = "0.1", optional = true } quick-error = { version = "1.2", optional = true } From 4ac51e097e59e2abe7fa11d7d362023b35c90b67 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Thu, 29 Nov 2018 16:54:56 -0800 Subject: [PATCH 428/453] update `buf_redux` --- multipart/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 2b89343ad..9b56e26e4 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -27,7 +27,7 @@ tempdir = "0.3" clippy = { version = ">=0.0, <0.1", optional = true} #Server Dependencies -buf_redux = { version = "0.7", optional = true, default-features = false } +buf_redux = { version = "0.8", optional = true, default-features = false } httparse = { version = "1.2", optional = true } twoway = { version = "0.1", optional = true } quick-error = { version = "1.2", optional = true } From 923cd30fc549a4cacaaf6f5ae3a2762f759462ba Mon Sep 17 00:00:00 2001 From: Pascal Seitz Date: Sat, 8 Dec 2018 21:08:15 +0100 Subject: [PATCH 429/453] switch to rocket 0.4 --- multipart/.travis.yml | 2 +- multipart/Cargo.toml | 5 ++--- multipart/examples/README.md | 2 +- multipart/examples/rocket.rs | 5 +++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/multipart/.travis.yml b/multipart/.travis.yml index 9d7f2df96..09f07e813 100644 --- a/multipart/.travis.yml +++ b/multipart/.travis.yml @@ -22,7 +22,7 @@ matrix: - rust: stable env: ARGS+=--features "use_arc_str" - rust: nightly - env: ARGS+=--features "nightly,rocket,rocket_codegen" + env: ARGS+=--features "nightly,rocket" script: - | if [ ${TRAVIS_RUST_VERSION} = "1.22.1" ]; then diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 9b56e26e4..0e6271353 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -39,8 +39,7 @@ tiny_http = { version = "0.6", optional = true } nickel = { version = ">=0.10.1", optional = true } # Only for Rocket example but dev-dependencies can't be optional -rocket = { version = "0.3", optional = true } -rocket_codegen = { version = "0.3", optional = true } +rocket = { version = "0.4", optional = true } [dev-dependencies] env_logger = "0.5" @@ -88,7 +87,7 @@ required-features = ["mock", "tiny_http", "server"] [[example]] name = "rocket" -required-features = ["mock", "rocket", "rocket_codegen", "server"] +required-features = ["mock", "rocket", "server"] [[bin]] name = "form_test" diff --git a/multipart/examples/README.md b/multipart/examples/README.md index ebad14068..5ebf96ebc 100644 --- a/multipart/examples/README.md +++ b/multipart/examples/README.md @@ -105,7 +105,7 @@ explicit support (the Rocket folks seem to want to handle `multipart/form-data` but haven't gotten around to implementing it yet; this would supercede any integration from `multipart`). ``` -$ cargo run --example rocket --features "rocket,rocket_codegen" +$ cargo run --example rocket --features "rocket" ``` [iamsebastian]: https://github.com/iamsebastian diff --git a/multipart/examples/rocket.rs b/multipart/examples/rocket.rs index fc518c9d5..cf9e6f714 100644 --- a/multipart/examples/rocket.rs +++ b/multipart/examples/rocket.rs @@ -2,10 +2,11 @@ // // Direct integration is not provided at this time as it appears the Rocket folks would prefer // to handle multipart requests behind the scenes. -#![feature(plugin)] -#![plugin(rocket_codegen)] +#![feature(proc_macro_hygiene, decl_macro)] +#![feature(plugin, custom_attribute)] extern crate multipart; +#[macro_use] extern crate rocket; use multipart::mock::StdoutTee; From a85fadd6360697fa5c9a14798f660c710a61e748 Mon Sep 17 00:00:00 2001 From: "Chris West (Faux)" Date: Sat, 15 Dec 2018 20:36:36 +0000 Subject: [PATCH 430/453] force lazy_static under 1.2 for 1.22.1 support --- multipart/.travis.yml | 8 +------- multipart/Cargo.toml | 2 ++ multipart/README.md | 5 +---- 3 files changed, 4 insertions(+), 11 deletions(-) diff --git a/multipart/.travis.yml b/multipart/.travis.yml index 09f07e813..12ab63ecc 100644 --- a/multipart/.travis.yml +++ b/multipart/.travis.yml @@ -4,7 +4,7 @@ branches: except: - fuzzing rust: - - 1.26.1 + - 1.22.1 - stable - beta - nightly @@ -16,7 +16,6 @@ env: - RUST_LOG=multipart=trace RUST_BACKTRACE=1 ARGS= matrix: include: - - rust: 1.22.1 - rust: stable env: ARGS+=--no-default-features --features "nickel" - rust: stable @@ -24,11 +23,6 @@ matrix: - rust: nightly env: ARGS+=--features "nightly,rocket" script: - - | - if [ ${TRAVIS_RUST_VERSION} = "1.22.1" ]; then - # we can't test because `env_logger` uses `lazy_static 1.2.0` which needs Rust 1.26 - cargo build --no-default-features --features "mock,client,server" && exit 0; - fi - cargo build --verbose $ARGS; - cargo test --verbose $ARGS -- --test-threads=1; - cargo doc --verbose $ARGS; diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 0e6271353..48db12d47 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -18,6 +18,8 @@ license = "MIT OR Apache-2.0" readme = "README.md" [dependencies] +# lazy-static (pulled in by e.g. nickel, regex) is incompatible with 1.22.1 +lazy_static = { version = ">=1.0,<1.2.0", optional = true } log = "0.4" mime = "0.2" mime_guess = "1.8" diff --git a/multipart/README.md b/multipart/README.md index aab85efc2..248b44e07 100644 --- a/multipart/README.md +++ b/multipart/README.md @@ -5,10 +5,7 @@ Client- and server-side abstractions for HTTP file uploads (POST requests with Supports several different (**sync**hronous API) HTTP crates. **Async**hronous (i.e. `futures`-based) API support will be provided by [multipart-async]. -Minimum supported Rust version: 1.22.1* -* only `mock`, `client` and `server` features, only guaranteed to compile - -Fully tested Rust version: 1.26.1 +Minimum supported Rust version: 1.22.1 ### [Documentation](http://docs.rs/multipart/) From 99d73bb1c8162c1454ec370154d255f15460e03b Mon Sep 17 00:00:00 2001 From: "Chris West (Faux)" Date: Sat, 15 Dec 2018 20:36:40 +0000 Subject: [PATCH 431/453] bump rand --- multipart/Cargo.toml | 2 +- multipart/src/lib.rs | 2 +- multipart/src/local_test.rs | 6 ++++-- multipart/src/mock.rs | 3 ++- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 48db12d47..25d1d9189 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -23,7 +23,7 @@ lazy_static = { version = ">=1.0,<1.2.0", optional = true } log = "0.4" mime = "0.2" mime_guess = "1.8" -rand = "0.4" +rand = "0.6" safemem = { version = "0.3", optional = true } tempdir = "0.3" clippy = { version = ">=0.0, <0.1", optional = true} diff --git a/multipart/src/lib.rs b/multipart/src/lib.rs index 0b04711b7..d5e8d5c1e 100644 --- a/multipart/src/lib.rs +++ b/multipart/src/lib.rs @@ -119,7 +119,7 @@ pub mod server; mod local_test; fn random_alphanumeric(len: usize) -> String { - rand::thread_rng().gen_ascii_chars().take(len).collect() + rand::thread_rng().sample_iter(&rand::distributions::Alphanumeric).take(len).collect() } #[cfg(test)] diff --git a/multipart/src/local_test.rs b/multipart/src/local_test.rs index 935a0462f..a1f16cdb2 100644 --- a/multipart/src/local_test.rs +++ b/multipart/src/local_test.rs @@ -301,6 +301,8 @@ fn gen_bool() -> bool { } fn gen_string() -> String { + use rand::distributions::Alphanumeric; + let mut rng_1 = rand::thread_rng(); let mut rng_2 = rand::thread_rng(); @@ -308,9 +310,9 @@ fn gen_string() -> String { let str_len_2 = rng_2.gen_range(MIN_LEN, MAX_LEN + 1); let num_dashes = rng_1.gen_range(0, MAX_DASHES + 1); - rng_1.gen_ascii_chars().take(str_len_1) + rng_1.sample_iter(&Alphanumeric).take(str_len_1) .chain(iter::repeat('-').take(num_dashes)) - .chain(rng_2.gen_ascii_chars().take(str_len_2)) + .chain(rng_2.sample_iter(&Alphanumeric).take(str_len_2)) .collect() } diff --git a/multipart/src/mock.rs b/multipart/src/mock.rs index f47dedeba..f6ea3bad5 100644 --- a/multipart/src/mock.rs +++ b/multipart/src/mock.rs @@ -9,7 +9,8 @@ use std::cell::{Cell, RefCell}; use std::io::{self, Read, Write}; use std::{fmt, thread}; -use rand::{self, Rng, ThreadRng}; +use rand::{self, Rng}; +use rand::prelude::ThreadRng; /// A mock implementation of `client::HttpRequest` which can spawn an `HttpBuffer`. /// From a6450207fe50225a1df9f98d7ecb01737390af1f Mon Sep 17 00:00:00 2001 From: Erick Tryzelaar Date: Fri, 18 Jan 2019 16:00:41 -0800 Subject: [PATCH 432/453] Use lazy_static 1.2.0, remove twoway/pcmp and require rust 1.24.1+ Before this patch, multipart got into an impossible sitation with it's dependencies. It errs with: ``` error: failed to select a version for `lazy_static`. ... required by package `multipart v0.15.4` versions that meet the requirements `>= 1.0, < 1.2.0` are: 1.1.0, 1.0.2, 1.0.1, 1.0.0 all possible versions conflict with previously selected packages. previously selected package `lazy_static v1.2.0` ... which is depended on by `ring v0.13.5` ... which is depended on by `cookie v0.11.0` ... which is depended on by `rocket_http v0.4.0` ... which is depended on by `rocket v0.4.0` ... which is depended on by `multipart v0.15.4 ``` This is due to ring 0.13.3 bumping lazy_static to 1.2.0 to avoid a [soundness bug](https://github.com/rust-lang-nursery/lazy-static.rs/issues/117). This patch fixes this problem by requiring at least rust 1.24.1. In addition, I noticed that the feature sse4 was depending on `twoway/pcmp`, but that has been [removed](https://github.com/bluss/twoway/pull/8). --- multipart/.travis.yml | 2 +- multipart/Cargo.toml | 5 ++--- multipart/README.md | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/multipart/.travis.yml b/multipart/.travis.yml index 12ab63ecc..1e72fda66 100644 --- a/multipart/.travis.yml +++ b/multipart/.travis.yml @@ -4,7 +4,7 @@ branches: except: - fuzzing rust: - - 1.22.1 + - 1.24.1 - stable - beta - nightly diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 25d1d9189..de0d36c28 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -18,8 +18,7 @@ license = "MIT OR Apache-2.0" readme = "README.md" [dependencies] -# lazy-static (pulled in by e.g. nickel, regex) is incompatible with 1.22.1 -lazy_static = { version = ">=1.0,<1.2.0", optional = true } +lazy_static = { version = "1.2.0", optional = true } log = "0.4" mime = "0.2" mime_guess = "1.8" @@ -55,7 +54,7 @@ nightly = [] bench = [] # Use this to enable SSE4.2 instructions in boundary finding # TODO: Benchmark this -sse4 = ["nightly", "twoway/pcmp"] +sse4 = ["nightly"] # switch uses of `Arc` for `Arc` (`From` impl only stabilized in 1.21) use_arc_str = [] diff --git a/multipart/README.md b/multipart/README.md index 248b44e07..4e89435ac 100644 --- a/multipart/README.md +++ b/multipart/README.md @@ -5,7 +5,7 @@ Client- and server-side abstractions for HTTP file uploads (POST requests with Supports several different (**sync**hronous API) HTTP crates. **Async**hronous (i.e. `futures`-based) API support will be provided by [multipart-async]. -Minimum supported Rust version: 1.22.1 +Minimum supported Rust version: 1.24.1 ### [Documentation](http://docs.rs/multipart/) From 278ce3059f43c30dc668dbef98e52041442f8a14 Mon Sep 17 00:00:00 2001 From: Erick Tryzelaar Date: Mon, 22 Oct 2018 17:11:49 -0700 Subject: [PATCH 433/453] Switch from tempdir to tempfile According to the tempdir repository, tempdir has been deprecated and replaced with tempfile: https://github.com/rust-lang-deprecated/tempdir#deprecation-note The current api publicly exposes the `tempdir::TempDir` type, so this patch is a breaking change to the public api, so I bumped the major version in the Cargo.toml file. --- multipart/Cargo.toml | 2 +- multipart/src/lib.rs | 4 ++-- multipart/src/server/iron.rs | 5 +++-- multipart/src/server/save.rs | 5 +++-- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index de0d36c28..0927d9853 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -24,7 +24,7 @@ mime = "0.2" mime_guess = "1.8" rand = "0.6" safemem = { version = "0.3", optional = true } -tempdir = "0.3" +tempfile = "3" clippy = { version = ">=0.0, <0.1", optional = true} #Server Dependencies diff --git a/multipart/src/lib.rs b/multipart/src/lib.rs index d5e8d5c1e..4f079e771 100644 --- a/multipart/src/lib.rs +++ b/multipart/src/lib.rs @@ -49,7 +49,7 @@ extern crate log; extern crate mime; extern crate mime_guess; extern crate rand; -extern crate tempdir; +extern crate tempfile; #[cfg(feature = "quick-error")] #[macro_use] @@ -125,4 +125,4 @@ fn random_alphanumeric(len: usize) -> String { #[cfg(test)] fn init_log() { let _ = env_logger::try_init(); -} \ No newline at end of file +} diff --git a/multipart/src/server/iron.rs b/multipart/src/server/iron.rs index 898ab5dd6..0d41431c9 100644 --- a/multipart/src/server/iron.rs +++ b/multipart/src/server/iron.rs @@ -11,6 +11,7 @@ use iron::{BeforeMiddleware, IronError, IronResult}; use std::path::PathBuf; use std::{error, fmt, io}; +use tempfile; use super::{FieldHeaders, HttpRequest, Multipart}; use super::save::{Entries, PartialReason, TempDir}; @@ -126,8 +127,8 @@ impl Intercept { let tempdir = self.temp_dir_path.as_ref() .map_or_else( - || TempDir::new("multipart-iron"), - |path| TempDir::new_in(path, "multipart-iron") + || tempfile::Builder::new().prefix("multipart-iron").tempdir(), + |path| tempfile::Builder::new().prefix("multipart-iron").tempdir_in(path) ) .map_err(|e| io_to_iron(e, "Error opening temporary directory for request."))?; diff --git a/multipart/src/server/save.rs b/multipart/src/server/save.rs index 0c0a03748..e9044bda7 100644 --- a/multipart/src/server/save.rs +++ b/multipart/src/server/save.rs @@ -8,13 +8,14 @@ pub use server::buf_redux::BufReader; -pub use tempdir::TempDir; +pub use tempfile::TempDir; use std::collections::HashMap; use std::io::prelude::*; use std::fs::{self, File, OpenOptions}; use std::path::{Path, PathBuf}; use std::{cmp, env, io, mem, str, u32, u64}; +use tempfile; use server::field::{FieldHeaders, MultipartField, MultipartData, ReadEntry, ReadEntryResult}; use server::ArcStr; @@ -247,7 +248,7 @@ impl SaveBuilder where M: ReadEntry { /// ### Note: Temporary /// See `SaveDir` for more info (the type of `Entries::save_dir`). pub fn temp_with_prefix(self, prefix: &str) -> EntriesSaveResult { - match TempDir::new(prefix) { + match tempfile::Builder::new().prefix(prefix).tempdir() { Ok(tempdir) => self.with_temp_dir(tempdir), Err(e) => SaveResult::Error(e), } From c5e77a6fc24d471ea8a63c663c8eadbc092b512b Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sun, 27 Jan 2019 23:36:06 -0800 Subject: [PATCH 434/453] drop `sse4`, `use_arc_str` features --- multipart/Cargo.toml | 5 ----- multipart/src/server/field.rs | 2 +- multipart/src/server/mod.rs | 14 -------------- 3 files changed, 1 insertion(+), 20 deletions(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 0927d9853..75a092b36 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -52,11 +52,6 @@ server = ["buf_redux", "httparse", "quick-error", "safemem", "twoway"] mock = [] nightly = [] bench = [] -# Use this to enable SSE4.2 instructions in boundary finding -# TODO: Benchmark this -sse4 = ["nightly"] -# switch uses of `Arc` for `Arc` (`From` impl only stabilized in 1.21) -use_arc_str = [] [[example]] name = "hyper_client" diff --git a/multipart/src/server/field.rs b/multipart/src/server/field.rs index 32222c743..b2e6186ba 100644 --- a/multipart/src/server/field.rs +++ b/multipart/src/server/field.rs @@ -114,7 +114,7 @@ fn copy_headers<'h, 'b: 'h>(raw: &[Header<'b>], headers: &'h mut [StrHeader<'b>] #[derive(Clone, Debug)] pub struct FieldHeaders { /// The field's name from the form. - pub name: ArcStr, + pub name: Arc, /// The filename of this entry, if supplied. This is not guaranteed to match the original file /// or even to be a valid filename for the current platform. diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index c9900e365..d29df21e0 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -30,20 +30,6 @@ use self::save::SaveBuilder; pub use self::save::{Entries, SaveResult, SavedField}; -/// Default typedef for shared strings. -/// -/// Enable the `use_arc_str` feature to use `Arc` instead, which saves an indirection but -/// cannot be constructed in Rust versions older than 1.21 (the `From` impl was stabilized -/// in that release). -#[cfg(not(feature = "use_arc_str"))] -pub type ArcStr = Arc; - -/// Optimized typedef for shared strings, replacing `Arc`. -/// -/// Enabled with the `use_arc_str` feature. -#[cfg(feature = "use_arc_str")] -pub type ArcStr = Arc; - macro_rules! try_opt ( ($expr:expr) => ( match $expr { From b8450822439d19cfd3d475bd69b34fe81201c3a5 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sun, 27 Jan 2019 23:37:15 -0800 Subject: [PATCH 435/453] remove `sse4` note in README --- multipart/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/multipart/README.md b/multipart/README.md index 4e89435ac..1845627c9 100644 --- a/multipart/README.md +++ b/multipart/README.md @@ -60,8 +60,8 @@ Fast, zero-copy HTTP header parsing, used to read field headers in `multipart/fo ### [twoway ![](https://img.shields.io/crates/v/twoway.svg)](https://crates.io/crates/twoway) -Fast string and byte-string search. Used to find boundaries in the request body. SSE 4.2 acceleration available -under the `sse42` or `twoway/pcmp` features. +Fast string and byte-string search. Used to find boundaries in the request body. Uses SIMD acceleration +when possible. ## License From 302dbefcc069c7311391cda027a666405f57cf77 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sun, 27 Jan 2019 23:53:57 -0800 Subject: [PATCH 436/453] fix lint errors --- multipart/src/bin/form_test.rs | 5 +++-- multipart/src/lib.rs | 2 +- multipart/src/local_test.rs | 7 ++++--- multipart/src/mock.rs | 3 +-- multipart/src/server/field.rs | 10 ++-------- multipart/src/server/mod.rs | 7 ++----- multipart/src/server/save.rs | 4 ++-- 7 files changed, 15 insertions(+), 23 deletions(-) diff --git a/multipart/src/bin/form_test.rs b/multipart/src/bin/form_test.rs index 348797642..8cf1b9499 100644 --- a/multipart/src/bin/form_test.rs +++ b/multipart/src/bin/form_test.rs @@ -18,8 +18,9 @@ fn main() { fn read_multipart(req: Request, mut resp: Response) { if let Ok(mut multipart) = Multipart::from_request(req) { - multipart.foreach_entry(|_| {}) - .map_err(|e| println!("error handling field: {}", e)); + if let Err(e) = multipart.foreach_entry(|_| {}) { + println!("error handling field: {}", e); + } } let mut file = File::open("src/bin/test_form.html") diff --git a/multipart/src/lib.rs b/multipart/src/lib.rs index 4f079e771..4673216fe 100644 --- a/multipart/src/lib.rs +++ b/multipart/src/lib.rs @@ -45,7 +45,7 @@ #[macro_use] extern crate log; -#[macro_use] +#[cfg_attr(test, macro_use)] extern crate mime; extern crate mime_guess; extern crate rand; diff --git a/multipart/src/local_test.rs b/multipart/src/local_test.rs index a1f16cdb2..796bc504a 100644 --- a/multipart/src/local_test.rs +++ b/multipart/src/local_test.rs @@ -8,9 +8,10 @@ use mock::{ClientRequest, HttpBuffer}; use server::{MultipartField, ReadEntry, FieldHeaders}; -use mime::{self, Mime}; +use mime::Mime; use rand::{self, Rng}; +use rand::seq::SliceRandom; use std::collections::{HashMap, HashSet}; use std::collections::hash_map::{Entry, OccupiedEntry}; @@ -431,11 +432,11 @@ fn test_server_entry_api(buf: HttpBuffer, fields: &mut TestFields) { } fn rand_mime() -> Mime { - rand::thread_rng().choose(&[ + [ // TODO: fill this out, preferably with variants that may be hard to parse // i.e. containing hyphens, mainly mime!(Application/OctetStream), mime!(Text/Plain), mime!(Image/Png), - ]).unwrap().clone() + ].choose(&mut rand::thread_rng()).unwrap().clone() } diff --git a/multipart/src/mock.rs b/multipart/src/mock.rs index f6ea3bad5..7d58905f4 100644 --- a/multipart/src/mock.rs +++ b/multipart/src/mock.rs @@ -5,9 +5,8 @@ // http://opensource.org/licenses/MIT>, at your option. This file may not be // copied, modified, or distributed except according to those terms. //! Mocked types for client-side and server-side APIs. -use std::cell::{Cell, RefCell}; use std::io::{self, Read, Write}; -use std::{fmt, thread}; +use std::fmt; use rand::{self, Rng}; use rand::prelude::ThreadRng; diff --git a/multipart/src/server/field.rs b/multipart/src/server/field.rs index b2e6186ba..c46edb879 100644 --- a/multipart/src/server/field.rs +++ b/multipart/src/server/field.rs @@ -6,18 +6,13 @@ // copied, modified, or distributed except according to those terms. //! `multipart` field header parsing. -use mime::{Mime, TopLevel, SubLevel}; - -use quick_error::ResultExt; +use mime::{Mime, TopLevel}; use std::error::Error; use std::io::{self, Read, BufRead}; use std::{str, fmt}; -// The AsciiExt import is needed for Rust older than 1.23.0. These two lines can -// be removed when supporting older Rust is no longer needed. -#[allow(deprecated, unused_imports)] -use std::ascii::AsciiExt; +use std::sync::Arc; use super::httparse::{self, EMPTY_HEADER, Header, Status, Error as HttparseError}; @@ -25,7 +20,6 @@ use self::ReadEntryResult::*; use super::save::SaveBuilder; -use super::ArcStr; const EMPTY_STR_HEADER: StrHeader<'static> = StrHeader { name: "", diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index d29df21e0..8cd01a0b8 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -17,7 +17,6 @@ extern crate twoway; use std::borrow::Borrow; use std::io::prelude::*; -use std::sync::Arc; use std::io; use self::boundary::BoundaryReader; @@ -225,8 +224,6 @@ fn issue_114() { ::init_log(); fn consume_all(mut rdr: R) { - let mut consume = 0; - loop { let consume = rdr.fill_buf().unwrap().len(); if consume == 0 { return; } @@ -255,11 +252,11 @@ fn issue_114() { multipart.foreach_entry(|_entry| { /* do nothing */}).unwrap(); // a different error if you skip the first field - multipart.foreach_entry(|mut entry| if *entry.headers.name != "key1" { consume_all(entry.data); }) + multipart.foreach_entry(|entry| if *entry.headers.name != *"key1" { consume_all(entry.data); }) .unwrap(); - multipart.foreach_entry(|mut entry| () /* match entry.headers.name.as_str() { + multipart.foreach_entry(|_entry| () /* match entry.headers.name.as_str() { "file" => { let mut vec = Vec::new(); entry.data.read_to_end(&mut vec).expect("can't read"); diff --git a/multipart/src/server/save.rs b/multipart/src/server/save.rs index e9044bda7..fa95c9ed8 100644 --- a/multipart/src/server/save.rs +++ b/multipart/src/server/save.rs @@ -14,11 +14,11 @@ use std::collections::HashMap; use std::io::prelude::*; use std::fs::{self, File, OpenOptions}; use std::path::{Path, PathBuf}; +use std::sync::Arc; use std::{cmp, env, io, mem, str, u32, u64}; use tempfile; use server::field::{FieldHeaders, MultipartField, MultipartData, ReadEntry, ReadEntryResult}; -use server::ArcStr; use self::SaveResult::*; use self::TextPolicy::*; @@ -616,7 +616,7 @@ pub struct Entries { /// Each vector is guaranteed not to be empty unless externally modified. // Even though individual fields might only have one entry, it's better to limit the // size of a value type in `HashMap` to improve cache efficiency in lookups. - pub fields: HashMap>, + pub fields: HashMap, Vec>, /// The directory that the entries in `fields` were saved into. pub save_dir: SaveDir, fields_count: u32, From 33ac1b79be422cb8cde2cd9acc07b640529aee4a Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Mon, 28 Jan 2019 14:25:19 -0800 Subject: [PATCH 437/453] cargo version 0.16.0 --- multipart/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 75a092b36..d73a9f7f9 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "multipart" -version = "0.15.4" +version = "0.16.0" authors = ["Austin Bonander "] From effbd57fb1b2e4e2baf6b33d7cdbc97a8f42d215 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Mon, 28 Jan 2019 14:33:09 -0800 Subject: [PATCH 438/453] client: add note about trailing CRLF --- multipart/src/client/mod.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/multipart/src/client/mod.rs b/multipart/src/client/mod.rs index 6446be29a..16141aad7 100644 --- a/multipart/src/client/mod.rs +++ b/multipart/src/client/mod.rs @@ -257,6 +257,8 @@ impl<'a, W: Write> MultipartWriter<'a, W> { } // always write the closing boundary, even for empty bodies + // trailing CRLF is optional but Actix requires it due to a naive implementation: + // https://github.com/actix/actix-web/issues/598 write!(self.inner, "--{}--\r\n", self.boundary)?; Ok(self.inner) } From e5ab1ae91895d07a36fcee040ff73f6de87d99a8 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Mon, 28 Jan 2019 14:33:29 -0800 Subject: [PATCH 439/453] cargo version 0.16.1 --- multipart/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index d73a9f7f9..ef24ca304 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "multipart" -version = "0.16.0" +version = "0.16.1" authors = ["Austin Bonander "] From 8bc74ab52423577cf0c4a12f95e027625deacb47 Mon Sep 17 00:00:00 2001 From: Bachue Zhou Date: Tue, 10 Sep 2019 17:42:25 +0800 Subject: [PATCH 440/453] upgrade mime & mime_guess --- multipart/.travis.yml | 2 +- multipart/Cargo.toml | 4 +- multipart/README.md | 2 +- multipart/src/client/lazy.rs | 175 ++++++++++++++++++++---------- multipart/src/client/mod.rs | 104 ++++++++++++------ multipart/src/lib.rs | 14 ++- multipart/src/local_test.rs | 160 ++++++++++++++++++--------- multipart/src/server/field.rs | 166 ++++++++++++++++++---------- multipart/src/server/tiny_http.rs | 17 ++- 9 files changed, 427 insertions(+), 217 deletions(-) diff --git a/multipart/.travis.yml b/multipart/.travis.yml index 1e72fda66..bf229ebb6 100644 --- a/multipart/.travis.yml +++ b/multipart/.travis.yml @@ -4,7 +4,7 @@ branches: except: - fuzzing rust: - - 1.24.1 + - 1.33.0 - stable - beta - nightly diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index ef24ca304..f77e1c391 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -20,8 +20,8 @@ readme = "README.md" [dependencies] lazy_static = { version = "1.2.0", optional = true } log = "0.4" -mime = "0.2" -mime_guess = "1.8" +mime = "0.3.14" +mime_guess = "2.0.1" rand = "0.6" safemem = { version = "0.3", optional = true } tempfile = "3" diff --git a/multipart/README.md b/multipart/README.md index 1845627c9..12b71e1a4 100644 --- a/multipart/README.md +++ b/multipart/README.md @@ -5,7 +5,7 @@ Client- and server-side abstractions for HTTP file uploads (POST requests with Supports several different (**sync**hronous API) HTTP crates. **Async**hronous (i.e. `futures`-based) API support will be provided by [multipart-async]. -Minimum supported Rust version: 1.24.1 +Minimum supported Rust version: 1.33.0 ### [Documentation](http://docs.rs/multipart/) diff --git a/multipart/src/client/lazy.rs b/multipart/src/client/lazy.rs index f3b708e94..4df40cfe5 100644 --- a/multipart/src/client/lazy.rs +++ b/multipart/src/client/lazy.rs @@ -83,7 +83,7 @@ impl<'a, E: Error> Error for LazyError<'a, E> { self.error.description() } - fn cause(&self) -> Option<&Error> { + fn cause(&self) -> Option<&dyn Error> { Some(&self.error) } } @@ -91,7 +91,10 @@ impl<'a, E: Error> Error for LazyError<'a, E> { impl<'a, E: fmt::Debug> fmt::Debug for LazyError<'a, E> { fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { if let Some(ref field_name) = self.field_name { - fmt.write_fmt(format_args!("LazyError (on field {:?}): {:?}", field_name, self.error)) + fmt.write_fmt(format_args!( + "LazyError (on field {:?}): {:?}", + field_name, self.error + )) } else { fmt.write_fmt(format_args!("LazyError (misc): {:?}", self.error)) } @@ -101,9 +104,15 @@ impl<'a, E: fmt::Debug> fmt::Debug for LazyError<'a, E> { impl<'a, E: fmt::Display> fmt::Display for LazyError<'a, E> { fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { if let Some(ref field_name) = self.field_name { - fmt.write_fmt(format_args!("Error writing field {:?}: {}", field_name, self.error)) + fmt.write_fmt(format_args!( + "Error writing field {:?}: {}", + field_name, self.error + )) } else { - fmt.write_fmt(format_args!("Error opening or flushing stream: {}", self.error)) + fmt.write_fmt(format_args!( + "Error opening or flushing stream: {}", + self.error + )) } } } @@ -121,19 +130,21 @@ pub struct Multipart<'n, 'd> { } impl<'n, 'd> Multipart<'n, 'd> { - /// Initialize a new lazy dynamic request. + /// Initialize a new lazy dynamic request. pub fn new() -> Self { Default::default() } - /// Add a text field to this request. - pub fn add_text(&mut self, name: N, text: T) -> &mut Self where N: Into>, T: Into> { - self.fields.push( - Field { - name: name.into(), - data: Data::Text(text.into()) - } - ); + /// Add a text field to this request. + pub fn add_text(&mut self, name: N, text: T) -> &mut Self + where + N: Into>, + T: Into>, + { + self.fields.push(Field { + name: name.into(), + data: Data::Text(text.into()), + }); self } @@ -142,29 +153,40 @@ impl<'n, 'd> Multipart<'n, 'd> { /// /// ### Note /// Does not check if `path` exists. - pub fn add_file(&mut self, name: N, path: P) -> &mut Self where N: Into>, P: IntoCowPath<'d> { - self.fields.push( - Field { - name: name.into(), - data: Data::File(path.into_cow_path()), - } - ); + pub fn add_file(&mut self, name: N, path: P) -> &mut Self + where + N: Into>, + P: IntoCowPath<'d>, + { + self.fields.push(Field { + name: name.into(), + data: Data::File(path.into_cow_path()), + }); self } /// Add a generic stream field to this request, - pub fn add_stream(&mut self, name: N, stream: R, filename: Option, mime: Option) -> &mut Self where N: Into>, R: Read + 'd, F: Into> { - self.fields.push( - Field { - name: name.into(), - data: Data::Stream(Stream { - content_type: mime.unwrap_or_else(::mime_guess::octet_stream), - filename: filename.map(|f| f.into()), - stream: Box::new(stream) - }), - } - ); + pub fn add_stream( + &mut self, + name: N, + stream: R, + filename: Option, + mime: Option, + ) -> &mut Self + where + N: Into>, + R: Read + 'd, + F: Into>, + { + self.fields.push(Field { + name: name.into(), + data: Data::Stream(Stream { + content_type: mime.unwrap_or(mime::APPLICATION_OCTET_STREAM), + filename: filename.map(|f| f.into()), + stream: Box::new(stream), + }), + }); self } @@ -173,7 +195,11 @@ impl<'n, 'd> Multipart<'n, 'd> { /// request, returning the response if successful, or the first error encountered. /// /// If any files were added by path they will now be opened for reading. - pub fn send(&mut self, mut req: R) -> Result<< R::Stream as HttpStream >::Response, LazyError<'n, < R::Stream as HttpStream >::Error>> { + pub fn send( + &mut self, + mut req: R, + ) -> Result<::Response, LazyError<'n, ::Error>> + { let mut prepared = self.prepare().map_err(LazyError::transform_err)?; req.apply_headers(prepared.boundary(), prepared.content_len()); @@ -219,7 +245,7 @@ impl<'n, 'd> fmt::Debug for Data<'n, 'd> { struct Stream<'n, 'd> { filename: Option>, content_type: Mime, - stream: Box, + stream: Box, } /// The result of [`Multipart::prepare()`](struct.Multipart.html#method.prepare). @@ -253,22 +279,29 @@ impl<'d> PreparedFields<'d> { for field in fields.drain(..) { match field.data { - Data::Text(text) => write!(text_data, "{}\r\nContent-Disposition: form-data; \ - name=\"{}\"\r\n\r\n{}", - boundary, field.name, text).unwrap(), + Data::Text(text) => write!( + text_data, + "{}\r\nContent-Disposition: form-data; \ + name=\"{}\"\r\n\r\n{}", + boundary, field.name, text + ) + .unwrap(), Data::File(file) => { let (stream, len) = PreparedField::from_path(field.name, &file, &boundary)?; content_len += len; streams.push(stream); - }, + } Data::Stream(stream) => { use_len = false; - streams.push( - PreparedField::from_stream(&field.name, &boundary, &stream.content_type, - stream.filename.as_ref().map(|f| &**f), - stream.stream)); - }, + streams.push(PreparedField::from_stream( + &field.name, + &boundary, + &stream.content_type, + stream.filename.as_ref().map(|f| &**f), + stream.stream, + )); + } } } @@ -285,7 +318,7 @@ impl<'d> PreparedFields<'d> { text_data: Cursor::new(text_data), streams, end_boundary: Cursor::new(boundary), - content_len: if use_len { Some(content_len) } else { None } , + content_len: if use_len { Some(content_len) } else { None }, }) } @@ -300,7 +333,7 @@ impl<'d> PreparedFields<'d> { let boundary = self.end_boundary.get_ref(); // Get just the bare boundary string - &boundary[4 .. boundary.len() - 2] + &boundary[4..boundary.len() - 2] } } @@ -313,7 +346,7 @@ impl<'d> Read for PreparedFields<'d> { let mut total_read = 0; - while total_read < buf.len() && !cursor_at_end(&self.end_boundary){ + while total_read < buf.len() && !cursor_at_end(&self.end_boundary) { let buf = &mut buf[total_read..]; total_read += if !cursor_at_end(&self.text_data) { @@ -324,7 +357,7 @@ impl<'d> Read for PreparedFields<'d> { res => { self.streams.push(field); res - }, + } }? } else { self.end_boundary.read(buf)? @@ -337,11 +370,15 @@ impl<'d> Read for PreparedFields<'d> { struct PreparedField<'d> { header: Cursor>, - stream: Box, + stream: Box, } impl<'d> PreparedField<'d> { - fn from_path<'n>(name: Cow<'n, str>, path: &Path, boundary: &str) -> Result<(Self, u64), LazyIoError<'n>> { + fn from_path<'n>( + name: Cow<'n, str>, + path: &Path, + boundary: &str, + ) -> Result<(Self, u64), LazyIoError<'n>> { let (content_type, filename) = super::mime_filename(&path); let file = try_lazy!(name, File::open(path)); @@ -354,11 +391,21 @@ impl<'d> PreparedField<'d> { Ok((stream, content_len)) } - fn from_stream(name: &str, boundary: &str, content_type: &Mime, filename: Option<&str>, stream: Box) -> Self { + fn from_stream( + name: &str, + boundary: &str, + content_type: &Mime, + filename: Option<&str>, + stream: Box, + ) -> Self { let mut header = Vec::new(); - write!(header, "{}\r\nContent-Disposition: form-data; name=\"{}\"", - boundary, name).unwrap(); + write!( + header, + "{}\r\nContent-Disposition: form-data; name=\"{}\"", + boundary, name + ) + .unwrap(); if let Some(filename) = filename { write!(header, "; filename=\"{}\"", filename).unwrap(); @@ -443,10 +490,14 @@ mod hyper { impl<'n, 'd> super::Multipart<'n, 'd> { /// #### Feature: `hyper` /// Complete a POST request with the given `hyper::client::Client` and URL. - /// + /// /// Supplies the fields in the body, optionally setting the content-length header if /// applicable (all added fields were text or files, i.e. no streams). - pub fn client_request(&mut self, client: &Client, url: U) -> HyperResult { + pub fn client_request( + &mut self, + client: &Client, + url: U, + ) -> HyperResult { self.client_request_mut(client, url, |r| r) } @@ -456,17 +507,20 @@ mod hyper { /// /// Note that the body, and the `ContentType` and `ContentLength` headers will be /// overwritten, either by this method or by Hyper. - pub fn client_request_mut RequestBuilder>(&mut self, client: &Client, url: U, - mut_fn: F) -> HyperResult { + pub fn client_request_mut RequestBuilder>( + &mut self, + client: &Client, + url: U, + mut_fn: F, + ) -> HyperResult { let mut fields = match self.prepare() { Ok(fields) => fields, Err(err) => { error!("Error preparing request: {}", err); return Err(err.error.into()); - }, + } }; - mut_fn(client.post(url)) .header(::client::hyper::content_type(fields.boundary())) .body(fields.to_body()) @@ -477,8 +531,11 @@ mod hyper { impl<'d> super::PreparedFields<'d> { /// #### Feature: `hyper` /// Convert `self` to `hyper::client::Body`. - #[cfg_attr(feature="clippy", warn(wrong_self_convention))] - pub fn to_body<'b>(&'b mut self) -> Body<'b> where 'd: 'b { + #[cfg_attr(feature = "clippy", warn(wrong_self_convention))] + pub fn to_body<'b>(&'b mut self) -> Body<'b> + where + 'd: 'b, + { if let Some(content_len) = self.content_len { Body::SizedBody(self, content_len) } else { diff --git a/multipart/src/client/mod.rs b/multipart/src/client/mod.rs index 16141aad7..3158d2a7d 100644 --- a/multipart/src/client/mod.rs +++ b/multipart/src/client/mod.rs @@ -28,12 +28,12 @@ pub use self::sized::SizedRequest; const BOUNDARY_LEN: usize = 16; macro_rules! map_self { - ($selff:expr, $try:expr) => ( + ($selff:expr, $try:expr) => { match $try { Ok(_) => Ok($selff), Err(err) => Err(err.into()), } - ) + }; } /// The entry point of the client-side multipart API. @@ -59,20 +59,24 @@ impl Multipart<()> { } } -impl Multipart { +impl Multipart { /// Write a text field to this multipart request. /// `name` and `val` can be either owned `String` or `&str`. /// /// ## Errors /// If something went wrong with the HTTP stream. - pub fn write_text, V: AsRef>(&mut self, name: N, val: V) -> Result<&mut Self, S::Error> { + pub fn write_text, V: AsRef>( + &mut self, + name: N, + val: V, + ) -> Result<&mut Self, S::Error> { map_self!(self, self.writer.write_text(name.as_ref(), val.as_ref())) } - - /// Open a file pointed to by `path` and write its contents to the multipart request, + + /// Open a file pointed to by `path` and write its contents to the multipart request, /// supplying its filename and guessing its `Content-Type` from its extension. /// - /// If you want to set these values manually, or use another type that implements `Read`, + /// If you want to set these values manually, or use another type that implements `Read`, /// use `.write_stream()`. /// /// `name` can be either `String` or `&str`, and `path` can be `PathBuf` or `&Path`. @@ -80,7 +84,11 @@ impl Multipart { /// ## Errors /// If there was a problem opening the file (was a directory or didn't exist), /// or if something went wrong with the HTTP stream. - pub fn write_file, P: AsRef>(&mut self, name: N, path: P) -> Result<&mut Self, S::Error> { + pub fn write_file, P: AsRef>( + &mut self, + name: N, + path: P, + ) -> Result<&mut Self, S::Error> { let name = name.as_ref(); let path = path.as_ref(); @@ -95,34 +103,47 @@ impl Multipart { /// /// ## Warning /// The given `Read` **must** be able to read to EOF (end of file/no more data), meaning - /// `Read::read()` returns `Ok(0)`. If it never returns EOF it will be read to infinity + /// `Read::read()` returns `Ok(0)`. If it never returns EOF it will be read to infinity /// and the request will never be completed. /// /// When using `SizedRequest` this also can cause out-of-control memory usage as the /// multipart data has to be written to an in-memory buffer so its size can be calculated. /// - /// Use `Read::take()` if you wish to send data from a `Read` + /// Use `Read::take()` if you wish to send data from a `Read` /// that will never return EOF otherwise. /// /// ## Errors /// If the reader returned an error, or if something went wrong with the HTTP stream. // RFC: How to format this declaration? pub fn write_stream, St: Read>( - &mut self, name: N, stream: &mut St, filename: Option<&str>, content_type: Option + &mut self, + name: N, + stream: &mut St, + filename: Option<&str>, + content_type: Option, ) -> Result<&mut Self, S::Error> { let name = name.as_ref(); - map_self!(self, self.writer.write_stream(stream, name, filename, content_type)) - } + map_self!( + self, + self.writer + .write_stream(stream, name, filename, content_type) + ) + } /// Finalize the request and return the response from the server, or the last error if set. pub fn send(self) -> Result { - self.writer.finish().map_err(io::Error::into).and_then(|body| body.finish()) - } + self.writer + .finish() + .map_err(io::Error::into) + .and_then(|body| body.finish()) + } } impl Multipart> -where ::Error: From { +where + ::Error: From, +{ /// Create a new `Multipart` using the `SizedRequest` wrapper around `req`. pub fn from_request_sized(req: R) -> Result { Multipart::from_request(SizedRequest::from_request(req)) @@ -134,18 +155,18 @@ pub trait HttpRequest { /// The HTTP stream type that can be opend by this request, to which the multipart data will be /// written. type Stream: HttpStream; - /// The error type for this request. + /// The error type for this request. /// Must be compatible with `io::Error` as well as `Self::HttpStream::Error` type Error: From + Into<::Error>; /// Set the `Content-Type` header to `multipart/form-data` and supply the `boundary` value. /// If `content_len` is given, set the `Content-Length` header to its value. - /// - /// Return `true` if any and all sanity checks passed and the stream is ready to be opened, + /// + /// Return `true` if any and all sanity checks passed and the stream is ready to be opened, /// or `false` otherwise. fn apply_headers(&mut self, boundary: &str, content_len: Option) -> bool; - /// Open the request stream and return it or any error otherwise. + /// Open the request stream and return it or any error otherwise. fn open_stream(self) -> Result; } @@ -157,7 +178,7 @@ pub trait HttpStream: Write { type Response; /// The error type for this stream. /// Must be compatible with `io::Error` as well as `Self::Request::Error`. - type Error: From + From<::Error>; + type Error: From + From<::Error>; /// Finalize and close the stream and return the response object, or any error otherwise. fn finish(self) -> Result; @@ -167,8 +188,12 @@ impl HttpRequest for () { type Stream = io::Sink; type Error = io::Error; - fn apply_headers(&mut self, _: &str, _: Option) -> bool { true } - fn open_stream(self) -> Result { Ok(io::sink()) } + fn apply_headers(&mut self, _: &str, _: Option) -> bool { + true + } + fn open_stream(self) -> Result { + Ok(io::sink()) + } } impl HttpStream for io::Sink { @@ -176,14 +201,19 @@ impl HttpStream for io::Sink { type Response = (); type Error = io::Error; - fn finish(self) -> Result { Ok(()) } + fn finish(self) -> Result { + Ok(()) + } } fn gen_boundary() -> String { ::random_alphanumeric(BOUNDARY_LEN) } -fn open_stream(mut req: R, content_len: Option) -> Result<(String, R::Stream), R::Error> { +fn open_stream( + mut req: R, + content_len: Option, +) -> Result<(String, R::Stream), R::Error> { let boundary = gen_boundary(); req.apply_headers(&boundary, content_len); req.open_stream().map(|stream| (boundary, stream)) @@ -225,19 +255,29 @@ impl<'a, W: Write> MultipartWriter<'a, W> { self.write_stream(&mut file, name, filename, Some(content_type)) } - fn write_stream(&mut self, stream: &mut S, name: &str, filename: Option<&str>, content_type: Option) -> io::Result<()> { + fn write_stream( + &mut self, + stream: &mut S, + name: &str, + filename: Option<&str>, + content_type: Option, + ) -> io::Result<()> { // This is necessary to make sure it is interpreted as a file on the server end. - let content_type = Some(content_type.unwrap_or_else(::mime_guess::octet_stream)); + let content_type = Some(content_type.unwrap_or(mime::APPLICATION_OCTET_STREAM)); chain_result! { self.write_field_headers(name, filename, content_type), io::copy(stream, &mut self.inner), - Ok(()) + Ok(()) } } - fn write_field_headers(&mut self, name: &str, filename: Option<&str>, content_type: Option) - -> io::Result<()> { + fn write_field_headers( + &mut self, + name: &str, + filename: Option<&str>, + content_type: Option, + ) -> io::Result<()> { chain_result! { // Write the first boundary, or the boundary for the previous field. self.write_boundary(), @@ -265,9 +305,9 @@ impl<'a, W: Write> MultipartWriter<'a, W> { } fn mime_filename(path: &Path) -> (Mime, Option<&str>) { - let content_type = ::mime_guess::guess_mime_type(path); + let content_type = ::mime_guess::from_path(path); let filename = opt_filename(path); - (content_type, filename) + (content_type.first_or_octet_stream(), filename) } fn opt_filename(path: &Path) -> Option<&str> { diff --git a/multipart/src/lib.rs b/multipart/src/lib.rs index 4673216fe..7ab4a5b0a 100644 --- a/multipart/src/lib.rs +++ b/multipart/src/lib.rs @@ -36,16 +36,15 @@ //! I have opened an issue as a place to collect responses and discussions for these questions //! [on Github](https://github.com/abonander/multipart/issues/96). Please quote the RFC-statement //! (and/or link to its source line) and provide your feedback there. -#![cfg_attr(feature="clippy", feature(plugin))] -#![cfg_attr(feature="clippy", plugin(clippy))] -#![cfg_attr(feature="clippy", deny(clippy))] +#![cfg_attr(feature = "clippy", feature(plugin))] +#![cfg_attr(feature = "clippy", plugin(clippy))] +#![cfg_attr(feature = "clippy", deny(clippy))] #![cfg_attr(feature = "bench", feature(test))] #![deny(missing_docs)] #[macro_use] extern crate log; -#[cfg_attr(test, macro_use)] extern crate mime; extern crate mime_guess; extern crate rand; @@ -87,7 +86,7 @@ use rand::Rng; /// Err(val) /// } /// } -/// +/// /// fn main() { /// let res = chain_result! { /// try_add_one(1), @@ -119,7 +118,10 @@ pub mod server; mod local_test; fn random_alphanumeric(len: usize) -> String { - rand::thread_rng().sample_iter(&rand::distributions::Alphanumeric).take(len).collect() + rand::thread_rng() + .sample_iter(&rand::distributions::Alphanumeric) + .take(len) + .collect() } #[cfg(test)] diff --git a/multipart/src/local_test.rs b/multipart/src/local_test.rs index 796bc504a..180d53de9 100644 --- a/multipart/src/local_test.rs +++ b/multipart/src/local_test.rs @@ -6,15 +6,15 @@ // copied, modified, or distributed except according to those terms. use mock::{ClientRequest, HttpBuffer}; -use server::{MultipartField, ReadEntry, FieldHeaders}; +use server::{FieldHeaders, MultipartField, ReadEntry}; use mime::Mime; -use rand::{self, Rng}; use rand::seq::SliceRandom; +use rand::{self, Rng}; -use std::collections::{HashMap, HashSet}; use std::collections::hash_map::{Entry, OccupiedEntry}; +use std::collections::{HashMap, HashSet}; use std::fmt; use std::io::prelude::*; use std::io::Cursor; @@ -28,8 +28,9 @@ const MAX_LEN: usize = 5; const MAX_DASHES: usize = 2; fn collect_rand, T, F: FnMut() -> T>(mut gen: F) -> C { - (0 .. rand::thread_rng().gen_range(MIN_FIELDS, MAX_FIELDS)) - .map(|_| gen()).collect() + (0..rand::thread_rng().gen_range(MIN_FIELDS, MAX_FIELDS)) + .map(|_| gen()) + .collect() } macro_rules! expect_fmt ( @@ -51,7 +52,10 @@ macro_rules! expect_ok_fmt ( ); ); -fn get_field<'m, V>(field: &FieldHeaders, fields: &'m mut HashMap) -> Option> { +fn get_field<'m, V>( + field: &FieldHeaders, + fields: &'m mut HashMap, +) -> Option> { match fields.entry(field.name.to_string()) { Entry::Occupied(occupied) => Some(occupied), Entry::Vacant(_) => None, @@ -75,14 +79,17 @@ impl TestFields { fn check_field(&mut self, mut field: MultipartField) -> M { // text/plain fields would be considered a file by `TestFields` if field.headers.content_type.is_none() { - let mut text_entries = expect_fmt!(get_field(&field.headers, &mut self.texts), - "Got text field that wasn't in original dataset: {:?}", - field.headers); + let mut text_entries = expect_fmt!( + get_field(&field.headers, &mut self.texts), + "Got text field that wasn't in original dataset: {:?}", + field.headers + ); let mut text = String::new(); expect_ok_fmt!( field.data.read_to_string(&mut text), - "error failed to read text data to string: {:?}\n{err}", field.headers + "error failed to read text data to string: {:?}\n{err}", + field.headers ); assert!( @@ -100,10 +107,11 @@ impl TestFields { return field.data.into_inner(); } - - let mut file_entries = expect_fmt!(get_field(&field.headers, &mut self.files), - "Got file field that wasn't in original dataset: {:?}", - field.headers); + let mut file_entries = expect_fmt!( + get_field(&field.headers, &mut self.files), + "Got file field that wasn't in original dataset: {:?}", + field.headers + ); let field_name = field.headers.name.clone(); let (test_entry, inner) = FileEntry::from_field(field); @@ -124,8 +132,16 @@ impl TestFields { } fn assert_is_empty(&self) { - assert!(self.texts.is_empty(), "Text Fields were not exhausted! {:?}", self.texts); - assert!(self.files.is_empty(), "File Fields were not exhausted! {:?}", self.files); + assert!( + self.texts.is_empty(), + "Text Fields were not exhausted! {:?}", + self.texts + ); + assert!( + self.files.is_empty(), + "File Fields were not exhausted! {:?}", + self.files + ); } } @@ -141,16 +157,20 @@ impl FileEntry { let mut data = Vec::new(); expect_ok_fmt!( field.data.read_to_end(&mut data), - "Error reading file field: {:?}\n{err}", field.headers + "Error reading file field: {:?}\n{err}", + field.headers ); ( FileEntry { - content_type: field.headers.content_type.unwrap_or(mime!(Application/OctetStream)), + content_type: field + .headers + .content_type + .unwrap_or(mime::APPLICATION_OCTET_STREAM), filename: field.headers.filename, data: PrintHex(data), }, - field.data.into_inner() + field.data.into_inner(), ) } @@ -249,7 +269,7 @@ fn lazy_client_entry_server() { } mod extended { - use super::{test_client, test_server, test_server_entry_api, test_client_lazy, TestFields}; + use super::{test_client, test_client_lazy, test_server, test_server_entry_api, TestFields}; use std::time::Instant; @@ -296,7 +316,6 @@ mod extended { } } - fn gen_bool() -> bool { rand::thread_rng().gen() } @@ -311,7 +330,9 @@ fn gen_string() -> String { let str_len_2 = rng_2.gen_range(MIN_LEN, MAX_LEN + 1); let num_dashes = rng_1.gen_range(0, MAX_DASHES + 1); - rng_1.sample_iter(&Alphanumeric).take(str_len_1) + rng_1 + .sample_iter(&Alphanumeric) + .take(str_len_1) .chain(iter::repeat('-').take(num_dashes)) .chain(rng_2.sample_iter(&Alphanumeric).take(str_len_2)) .collect() @@ -326,30 +347,44 @@ fn test_client(test_fields: &TestFields) -> HttpBuffer { let request = ClientRequest::default(); - let mut test_files = test_fields.files.iter().flat_map( - |(name, files)| files.iter().map(move |file| (name, file)) - ); + let mut test_files = test_fields + .files + .iter() + .flat_map(|(name, files)| files.iter().map(move |file| (name, file))); - let test_texts = test_fields.texts.iter().flat_map( - |(name, texts)| texts.iter().map(move |text| (name, text)) - ); + let test_texts = test_fields + .texts + .iter() + .flat_map(|(name, texts)| texts.iter().map(move |text| (name, text))); let mut multipart = Multipart::from_request(request).unwrap(); - + // Intersperse file fields amongst text fields for (name, text) in test_texts { if let Some((file_name, file)) = test_files.next() { - multipart.write_stream(file_name, &mut &*file.data.0, file.filename(), - Some(file.content_type.clone())).unwrap(); + multipart + .write_stream( + file_name, + &mut &*file.data.0, + file.filename(), + Some(file.content_type.clone()), + ) + .unwrap(); } - multipart.write_text(name, text).unwrap(); + multipart.write_text(name, text).unwrap(); } // Write remaining files for (file_name, file) in test_files { - multipart.write_stream(file_name, &mut &*file.data.0, file.filename(), - Some(file.content_type.clone())).unwrap(); + multipart + .write_stream( + file_name, + &mut &*file.data.0, + file.filename(), + Some(file.content_type.clone()), + ) + .unwrap(); } multipart.send().unwrap() @@ -360,26 +395,36 @@ fn test_client_lazy(test_fields: &TestFields) -> HttpBuffer { let mut multipart = Multipart::new(); - let mut test_files = test_fields.files.iter().flat_map( - |(name, files)| files.iter().map(move |file| (name, file)) - ); + let mut test_files = test_fields + .files + .iter() + .flat_map(|(name, files)| files.iter().map(move |file| (name, file))); - let test_texts = test_fields.texts.iter().flat_map( - |(name, texts)| texts.iter().map(move |text| (name, text)) - ); + let test_texts = test_fields + .texts + .iter() + .flat_map(|(name, texts)| texts.iter().map(move |text| (name, text))); for (name, text) in test_texts { if let Some((file_name, file)) = test_files.next() { - multipart.add_stream(&**file_name, Cursor::new(&file.data.0), file.filename(), - Some(file.content_type.clone())); + multipart.add_stream( + &**file_name, + Cursor::new(&file.data.0), + file.filename(), + Some(file.content_type.clone()), + ); } multipart.add_text(&**name, &**text); } for (file_name, file) in test_files { - multipart.add_stream(&**file_name, Cursor::new(&file.data.0), file.filename(), - Some(file.content_type.clone())); + multipart.add_stream( + &**file_name, + Cursor::new(&file.data.0), + file.filename(), + Some(file.content_type.clone()), + ); } let mut prepared = multipart.prepare().unwrap(); @@ -400,7 +445,10 @@ fn test_server(buf: HttpBuffer, fields: &mut TestFields) { let server_buf = buf.for_server(); if let Some(content_len) = server_buf.content_len { - assert!(content_len == server_buf.data.len() as u64, "Supplied content_len different from actual"); + assert!( + content_len == server_buf.data.len() as u64, + "Supplied content_len different from actual" + ); } let mut multipart = Multipart::from_request(server_buf) @@ -417,13 +465,18 @@ fn test_server_entry_api(buf: HttpBuffer, fields: &mut TestFields) { let server_buf = buf.for_server(); if let Some(content_len) = server_buf.content_len { - assert!(content_len == server_buf.data.len() as u64, "Supplied content_len different from actual"); + assert!( + content_len == server_buf.data.len() as u64, + "Supplied content_len different from actual" + ); } let mut multipart = Multipart::from_request(server_buf) .unwrap_or_else(|_| panic!("Buffer should be multipart!")); - let entry = multipart.into_entry().expect_alt("Expected entry, got none", "Error reading entry"); + let entry = multipart + .into_entry() + .expect_alt("Expected entry, got none", "Error reading entry"); multipart = fields.check_field(entry); while let Some(entry) = multipart.into_entry().unwrap_opt() { @@ -433,10 +486,11 @@ fn test_server_entry_api(buf: HttpBuffer, fields: &mut TestFields) { fn rand_mime() -> Mime { [ - // TODO: fill this out, preferably with variants that may be hard to parse - // i.e. containing hyphens, mainly - mime!(Application/OctetStream), - mime!(Text/Plain), - mime!(Image/Png), - ].choose(&mut rand::thread_rng()).unwrap().clone() + mime::APPLICATION_OCTET_STREAM, + mime::TEXT_PLAIN, + mime::IMAGE_PNG, + ] + .choose(&mut rand::thread_rng()) + .unwrap() + .clone() } diff --git a/multipart/src/server/field.rs b/multipart/src/server/field.rs index c46edb879..6ccf9525b 100644 --- a/multipart/src/server/field.rs +++ b/multipart/src/server/field.rs @@ -6,32 +6,29 @@ // copied, modified, or distributed except according to those terms. //! `multipart` field header parsing. -use mime::{Mime, TopLevel}; +use mime::Mime; use std::error::Error; -use std::io::{self, Read, BufRead}; -use std::{str, fmt}; +use std::io::{self, BufRead, Read}; +use std::{fmt, str}; use std::sync::Arc; -use super::httparse::{self, EMPTY_HEADER, Header, Status, Error as HttparseError}; +use super::httparse::{self, Error as HttparseError, Header, Status, EMPTY_HEADER}; use self::ReadEntryResult::*; use super::save::SaveBuilder; - -const EMPTY_STR_HEADER: StrHeader<'static> = StrHeader { - name: "", - val: "", -}; +const EMPTY_STR_HEADER: StrHeader<'static> = StrHeader { name: "", val: "" }; macro_rules! invalid_cont_disp { ($reason: expr, $cause: expr) => { - return Err( - ParseHeaderError::InvalidContDisp($reason, $cause.to_string()) - ); - } + return Err(ParseHeaderError::InvalidContDisp( + $reason, + $cause.to_string(), + )); + }; } /// Not exposed @@ -43,7 +40,7 @@ pub struct StrHeader<'a> { struct DisplayHeaders<'s, 'a: 's>(&'s [StrHeader<'a>]); -impl <'s, 'a: 's> fmt::Display for DisplayHeaders<'s, 'a> { +impl<'s, 'a: 's> fmt::Display for DisplayHeaders<'s, 'a> { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { for hdr in self.0 { writeln!(f, "{}: {}", hdr.name, hdr.val)?; @@ -54,7 +51,10 @@ impl <'s, 'a: 's> fmt::Display for DisplayHeaders<'s, 'a> { } fn with_headers(r: &mut R, closure: F) -> Result -where R: BufRead, F: FnOnce(&[StrHeader]) -> Ret { +where + R: BufRead, + F: FnOnce(&[StrHeader]) -> Ret, +{ const HEADER_LEN: usize = 4; let consume; @@ -83,7 +83,7 @@ where R: BufRead, F: FnOnce(&[StrHeader]) -> Ret { consume = consume_; ret = closure(headers); break; - }, + } } } @@ -91,7 +91,10 @@ where R: BufRead, F: FnOnce(&[StrHeader]) -> Ret { Ok(ret) } -fn copy_headers<'h, 'b: 'h>(raw: &[Header<'b>], headers: &'h mut [StrHeader<'b>]) -> io::Result<&'h [StrHeader<'b>]> { +fn copy_headers<'h, 'b: 'h>( + raw: &[Header<'b>], + headers: &'h mut [StrHeader<'b>], +) -> io::Result<&'h [StrHeader<'b>]> { for (raw, header) in raw.iter().zip(&mut *headers) { header.name = raw.name; header.val = io_str_utf8(raw.value)?; @@ -154,7 +157,7 @@ impl ContentDisp { header } else { return Err(ParseHeaderError::MissingContentDisposition( - DisplayHeaders(headers).to_string() + DisplayHeaders(headers).to_string(), )); }; @@ -167,25 +170,32 @@ impl ContentDisp { invalid_cont_disp!("unexpected Content-Disposition value", disp_type); } after_disp_type - }, - None => invalid_cont_disp!("expected additional data after Content-Disposition type", - header.val), + } + None => invalid_cont_disp!( + "expected additional data after Content-Disposition type", + header.val + ), }; // Content-Disposition: form-data; name=? let (field_name, filename) = match get_str_after("name=", ';', after_disp_type) { - None => invalid_cont_disp!("expected field name and maybe filename, got", - after_disp_type), + None => invalid_cont_disp!( + "expected field name and maybe filename, got", + after_disp_type + ), // Content-Disposition: form-data; name={field_name}; filename=? Some((field_name, after_field_name)) => { let field_name = trim_quotes(field_name); let filename = get_str_after("filename=", ';', after_field_name) .map(|(filename, _)| trim_quotes(filename).to_owned()); (field_name, filename) - }, + } }; - Ok(ContentDisp { field_name: field_name.to_owned(), filename }) + Ok(ContentDisp { + field_name: field_name.to_owned(), + filename, + }) } } @@ -193,8 +203,9 @@ fn parse_content_type(headers: &[StrHeader]) -> Result, ParseHeader if let Some(header) = find_header(headers, "Content-Type") { // Boundary parameter will be parsed into the `Mime` debug!("Found Content-Type: {:?}", header.val); - Ok(Some(header.val.parse::() - .map_err(|_| ParseHeaderError::MimeError(header.val.into()))?)) + Ok(Some(header.val.parse::().map_err(|_| { + ParseHeaderError::MimeError(header.val.into()) + })?)) } else { Ok(None) } @@ -225,7 +236,10 @@ impl MultipartField { /// /// Detecting character encodings by any means is (currently) beyond the scope of this crate. pub fn is_text(&self) -> bool { - self.headers.content_type.as_ref().map_or(true, |ct| ct.0 == TopLevel::Text) + self.headers + .content_type + .as_ref() + .map_or(true, |ct| ct.type_() == mime::TEXT) } /// Read the next entry in the request. @@ -237,18 +251,21 @@ impl MultipartField { /// /// Returns `Ok(Some(self))` if another entry was read, `Ok(None)` if the end of the body was /// reached, and `Err(e)` for any errors that occur. - pub fn next_entry_inplace(&mut self) -> io::Result> where for<'a> &'a mut M: ReadEntry { + pub fn next_entry_inplace(&mut self) -> io::Result> + where + for<'a> &'a mut M: ReadEntry, + { let multipart = self.data.take_inner(); match multipart.read_entry() { Entry(entry) => { *self = entry; Ok(Some(self)) - }, + } End(multipart) => { self.data.give_inner(multipart); Ok(None) - }, + } Error(multipart, err) => { self.data.give_inner(multipart); Err(err) @@ -271,7 +288,10 @@ const DATA_INNER_ERR: &str = "MultipartFile::inner taken and not replaced; this relevant backtrace and debug logs at \ https://github.com/abonander/multipart"; -impl MultipartData where M: ReadEntry { +impl MultipartData +where + M: ReadEntry, +{ /// Get a builder type which can save the field with or without a size limit. pub fn save(&mut self) -> SaveBuilder<&mut Self> { SaveBuilder::new(self) @@ -306,7 +326,7 @@ impl MultipartData where M: ReadEntry { } impl Read for MultipartData { - fn read(&mut self, buf: &mut [u8]) -> io::Result{ + fn read(&mut self, buf: &mut [u8]) -> io::Result { self.inner_mut().source_mut().read(buf) } } @@ -333,11 +353,19 @@ fn trim_quotes(s: &str) -> &str { } /// Get the string after `needle` in `haystack`, stopping before `end_val_delim` -fn get_str_after<'a>(needle: &str, end_val_delim: char, haystack: &'a str) -> Option<(&'a str, &'a str)> { +fn get_str_after<'a>( + needle: &str, + end_val_delim: char, + haystack: &'a str, +) -> Option<(&'a str, &'a str)> { let val_start_idx = try_opt!(haystack.find(needle)) + needle.len(); - let val_end_idx = haystack[val_start_idx..].find(end_val_delim) + let val_end_idx = haystack[val_start_idx..] + .find(end_val_delim) .map_or(haystack.len(), |end_idx| end_idx + val_start_idx); - Some((&haystack[val_start_idx..val_end_idx], &haystack[val_end_idx..])) + Some(( + &haystack[val_start_idx..val_end_idx], + &haystack[val_end_idx..], + )) } fn io_str_utf8(buf: &[u8]) -> io::Result<&str> { @@ -347,7 +375,9 @@ fn io_str_utf8(buf: &[u8]) -> io::Result<&str> { fn find_header<'a, 'b>(headers: &'a [StrHeader<'b>], name: &str) -> Option<&'a StrHeader<'b>> { // Field names are case insensitive and consist of ASCII characters // only (see https://tools.ietf.org/html/rfc822#section-3.2). - headers.iter().find(|header| header.name.eq_ignore_ascii_case(name)) + headers + .iter() + .find(|header| header.name.eq_ignore_ascii_case(name)) } /// Common trait for `Multipart` and `&mut Multipart` @@ -365,25 +395,23 @@ pub trait ReadEntry: PrivReadEntry + Sized { let field_headers: FieldHeaders = try_read_entry!(self; self.read_headers()); if let Some(ct) = field_headers.content_type.as_ref() { - if ct.0 == TopLevel::Multipart { + if ct.type_() == mime::MULTIPART { // fields of this type are sent by (supposedly) no known clients // (https://tools.ietf.org/html/rfc7578#appendix-A) so I'd be fascinated // to hear about any in the wild - info!("Found nested multipart field: {:?}:\r\n\ - Please report this client's User-Agent and any other available details \ - at https://github.com/abonander/multipart/issues/56", - field_headers); + info!( + "Found nested multipart field: {:?}:\r\n\ + Please report this client's User-Agent and any other available details \ + at https://github.com/abonander/multipart/issues/56", + field_headers + ); } } - Entry( - MultipartField { - headers: field_headers, - data: MultipartData { - inner: Some(self), - }, - } - ) + Entry(MultipartField { + headers: field_headers, + data: MultipartData { inner: Some(self) }, + }) } /// Equivalent to `read_entry()` but takes `&mut self` @@ -464,8 +492,10 @@ impl ReadEntryResult { /// Attempt to unwrap `Entry`, panicking if this is `End` or `Error`. pub fn unwrap(self) -> Entry { - self.expect_alt("`ReadEntryResult::unwrap()` called on `End` value", - "`ReadEntryResult::unwrap()` called on `Error` value: {:?}") + self.expect_alt( + "`ReadEntryResult::unwrap()` called on `End` value", + "`ReadEntryResult::unwrap()` called on `Error` value: {:?}", + ) } /// Attempt to unwrap `Entry`, panicking if this is `End` or `Error` @@ -542,12 +572,32 @@ quick_error! { #[test] fn test_find_header() { let headers = [ - StrHeader { name: "Content-Type", val: "text/plain" }, - StrHeader { name: "Content-disposition", val: "form-data" }, - StrHeader { name: "content-transfer-encoding", val: "binary" } + StrHeader { + name: "Content-Type", + val: "text/plain", + }, + StrHeader { + name: "Content-disposition", + val: "form-data", + }, + StrHeader { + name: "content-transfer-encoding", + val: "binary", + }, ]; - assert_eq!(find_header(&headers, "Content-Type").unwrap().val, "text/plain"); - assert_eq!(find_header(&headers, "Content-Disposition").unwrap().val, "form-data"); - assert_eq!(find_header(&headers, "Content-Transfer-Encoding").unwrap().val, "binary"); + assert_eq!( + find_header(&headers, "Content-Type").unwrap().val, + "text/plain" + ); + assert_eq!( + find_header(&headers, "Content-Disposition").unwrap().val, + "form-data" + ); + assert_eq!( + find_header(&headers, "Content-Transfer-Encoding") + .unwrap() + .val, + "binary" + ); } diff --git a/multipart/src/server/tiny_http.rs b/multipart/src/server/tiny_http.rs index b2eaa8594..ae246cf21 100644 --- a/multipart/src/server/tiny_http.rs +++ b/multipart/src/server/tiny_http.rs @@ -11,16 +11,23 @@ use super::HttpRequest; use std::io::Read; impl<'r> HttpRequest for &'r mut TinyHttpRequest { - type Body = &'r mut Read; - + type Body = &'r mut dyn Read; + fn multipart_boundary(&self) -> Option<&str> { const BOUNDARY: &str = "boundary="; - let content_type = try_opt!(self.headers().iter().find(|header| header.field.equiv("Content-Type"))).value.as_str(); + let content_type = try_opt!(self + .headers() + .iter() + .find(|header| header.field.equiv("Content-Type"))) + .value + .as_str(); let start = try_opt!(content_type.find(BOUNDARY)) + BOUNDARY.len(); - let end = content_type[start..].find(';').map_or(content_type.len(), |end| start + end); + let end = content_type[start..] + .find(';') + .map_or(content_type.len(), |end| start + end); - Some(&content_type[start .. end]) + Some(&content_type[start..end]) } fn body(self) -> Self::Body { From 1ecace6a4080856de425a6261041314f44129cfb Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Tue, 9 Jun 2020 13:37:58 -0700 Subject: [PATCH 441/453] bump version to 0.17.0 --- multipart/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index f77e1c391..5abba615c 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "multipart" -version = "0.16.1" +version = "0.17.0" authors = ["Austin Bonander "] From d3cbaa4b92c453558700b9a310d041db6d7e5780 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Tue, 9 Jun 2020 13:46:17 -0700 Subject: [PATCH 442/453] add maintenance note to README --- multipart/README.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/multipart/README.md b/multipart/README.md index 12b71e1a4..b1b3519b0 100644 --- a/multipart/README.md +++ b/multipart/README.md @@ -5,7 +5,16 @@ Client- and server-side abstractions for HTTP file uploads (POST requests with Supports several different (**sync**hronous API) HTTP crates. **Async**hronous (i.e. `futures`-based) API support will be provided by [multipart-async]. -Minimum supported Rust version: 1.33.0 +##### Minimum supported Rust version: 1.33.0 + +##### Maintenance Status: Passive + +As the web ecosystem in Rust moves towards asynchronous APIs, the need for this crate in synchronous +API form becomes dubious. This crate in its current form is usable enough, so as of June 2020 it +is now in passive maintenance mode; bug reports will be addressed as time permits and PRs will be +accepted but otherwise no new development of the existing API is taking place. + +Look for a release of [multipart-async] soon which targets newer releases of Hyper. ### [Documentation](http://docs.rs/multipart/) From 875b2250f4f608a275777fc5fced67676097fa63 Mon Sep 17 00:00:00 2001 From: Peter Cunderlik Date: Mon, 16 Nov 2020 17:08:03 +0000 Subject: [PATCH 443/453] Bump rand to 0.7 to address RUSTSEC-2019-0035 rand<0.7 depends on rand_core 0.3 which suffers from https://rustsec.org/advisories/RUSTSEC-2019-0035 --- multipart/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 5abba615c..e91b4456d 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -22,7 +22,7 @@ lazy_static = { version = "1.2.0", optional = true } log = "0.4" mime = "0.3.14" mime_guess = "2.0.1" -rand = "0.6" +rand = "0.7" safemem = { version = "0.3", optional = true } tempfile = "3" clippy = { version = ">=0.0, <0.1", optional = true} From 58db144e4b4ddae1d79d6fe5f7a33d18694be514 Mon Sep 17 00:00:00 2001 From: Austin Bonander Date: Sat, 9 Jan 2021 15:08:30 -0800 Subject: [PATCH 444/453] bump patch version --- multipart/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index e91b4456d..4165322e3 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "multipart" -version = "0.17.0" +version = "0.17.1" authors = ["Austin Bonander "] From 7f462a47dafe1a326fcae60be46282647283d296 Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Wed, 13 Jan 2021 15:55:28 -0600 Subject: [PATCH 445/453] Update rand to 0.8, bump MSRV --- multipart/Cargo.toml | 2 +- multipart/README.md | 2 +- multipart/src/lib.rs | 1 + multipart/src/local_test.rs | 11 ++++++----- multipart/src/mock.rs | 4 ++-- 5 files changed, 11 insertions(+), 9 deletions(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 4165322e3..b20019017 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -22,7 +22,7 @@ lazy_static = { version = "1.2.0", optional = true } log = "0.4" mime = "0.3.14" mime_guess = "2.0.1" -rand = "0.7" +rand = "0.8" safemem = { version = "0.3", optional = true } tempfile = "3" clippy = { version = ">=0.0, <0.1", optional = true} diff --git a/multipart/README.md b/multipart/README.md index b1b3519b0..a97f80581 100644 --- a/multipart/README.md +++ b/multipart/README.md @@ -5,7 +5,7 @@ Client- and server-side abstractions for HTTP file uploads (POST requests with Supports several different (**sync**hronous API) HTTP crates. **Async**hronous (i.e. `futures`-based) API support will be provided by [multipart-async]. -##### Minimum supported Rust version: 1.33.0 +##### Minimum supported Rust version: 1.36.0 ##### Maintenance Status: Passive diff --git a/multipart/src/lib.rs b/multipart/src/lib.rs index 7ab4a5b0a..896ea7549 100644 --- a/multipart/src/lib.rs +++ b/multipart/src/lib.rs @@ -121,6 +121,7 @@ fn random_alphanumeric(len: usize) -> String { rand::thread_rng() .sample_iter(&rand::distributions::Alphanumeric) .take(len) + .map(|c| c as char) .collect() } diff --git a/multipart/src/local_test.rs b/multipart/src/local_test.rs index 180d53de9..3d24a9204 100644 --- a/multipart/src/local_test.rs +++ b/multipart/src/local_test.rs @@ -28,7 +28,7 @@ const MAX_LEN: usize = 5; const MAX_DASHES: usize = 2; fn collect_rand, T, F: FnMut() -> T>(mut gen: F) -> C { - (0..rand::thread_rng().gen_range(MIN_FIELDS, MAX_FIELDS)) + (0..rand::thread_rng().gen_range(MIN_FIELDS..MAX_FIELDS)) .map(|_| gen()) .collect() } @@ -326,15 +326,16 @@ fn gen_string() -> String { let mut rng_1 = rand::thread_rng(); let mut rng_2 = rand::thread_rng(); - let str_len_1 = rng_1.gen_range(MIN_LEN, MAX_LEN + 1); - let str_len_2 = rng_2.gen_range(MIN_LEN, MAX_LEN + 1); - let num_dashes = rng_1.gen_range(0, MAX_DASHES + 1); + let str_len_1 = rng_1.gen_range(MIN_LEN..=MAX_LEN); + let str_len_2 = rng_2.gen_range(MIN_LEN..=MAX_LEN); + let num_dashes = rng_1.gen_range(0..=MAX_DASHES); rng_1 .sample_iter(&Alphanumeric) .take(str_len_1) - .chain(iter::repeat('-').take(num_dashes)) + .chain(iter::repeat(b'-').take(num_dashes)) .chain(rng_2.sample_iter(&Alphanumeric).take(str_len_2)) + .map(|c| c as char) .collect() } diff --git a/multipart/src/mock.rs b/multipart/src/mock.rs index 7d58905f4..2947661bd 100644 --- a/multipart/src/mock.rs +++ b/multipart/src/mock.rs @@ -92,7 +92,7 @@ impl Write for HttpBuffer { } // Simulate the randomness of a network connection by not always reading everything - let len = self.rng.gen_range(1, buf.len() + 1); + let len = self.rng.gen_range(1..=buf.len()); self.buf.write(&buf[..len]) } @@ -159,7 +159,7 @@ impl<'a> Read for ServerRequest<'a> { } // Simulate the randomness of a network connection by not always reading everything - let len = self.rng.gen_range(1, out.len() + 1); + let len = self.rng.gen_range(1..=out.len()); self.data.read(&mut out[..len]) } } From 1bb2da0ccf14a99e598dde457edb1e93146da4d6 Mon Sep 17 00:00:00 2001 From: Noah <33094578+coolreader18@users.noreply.github.com> Date: Wed, 13 Jan 2021 15:56:21 -0600 Subject: [PATCH 446/453] bump version to 0.18.0 --- multipart/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index b20019017..7516948cf 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "multipart" -version = "0.17.1" +version = "0.18.0" authors = ["Austin Bonander "] From 964e09d834d66b8c17d158e50fb922104772d945 Mon Sep 17 00:00:00 2001 From: Alex Touchet Date: Mon, 1 Mar 2021 17:19:25 -0800 Subject: [PATCH 447/453] Update iron URL --- multipart/examples/README.md | 2 +- multipart/src/server/iron.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/multipart/examples/README.md b/multipart/examples/README.md index 5ebf96ebc..580d3bbdc 100644 --- a/multipart/examples/README.md +++ b/multipart/examples/README.md @@ -46,7 +46,7 @@ $ cargo run --example hyper_server ----------------- Author: [White-Oak] -This example shows how to use `multipart` with the [Iron web application framework](http://ironframework.io/), via `multipart`'s support +This example shows how to use `multipart` with the [Iron web application framework](https://github.com/iron/iron), via `multipart`'s support for the `iron::Request` type. To run: diff --git a/multipart/src/server/iron.rs b/multipart/src/server/iron.rs index 0d41431c9..c4bef5712 100644 --- a/multipart/src/server/iron.rs +++ b/multipart/src/server/iron.rs @@ -1,4 +1,4 @@ -//! Integration with the [Iron](https://ironframework.io) framework, enabled with the `iron` feature (optional). Includes a `BeforeMiddleware` implementation. +//! Integration with the [Iron](https://github.com/iron/iron) framework, enabled with the `iron` feature (optional). Includes a `BeforeMiddleware` implementation. //! //! Not shown here: `impl `[`HttpRequest`](../trait.HttpRequest.html#implementors)` for //! iron::Request`. From 2604a43f2c848ad8d9cd579271a037da12a6a73e Mon Sep 17 00:00:00 2001 From: Trevor Gross Date: Sat, 14 Jun 2025 00:29:06 +0000 Subject: [PATCH 448/453] Resolve FCWs in multipart MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Apply Ondřej's patch from [1] to resolve future compatibility warnings in multipart, then update it further to resolve FCWs in all default features (the patch only resolved issues with the features that Rouille uses). Co-authored-by: Ondřej Hruška [1]: https://github.com/tomaka/rouille/issues/271 --- multipart/Cargo.toml | 4 ++++ multipart/src/client/lazy.rs | 3 ++- multipart/src/client/mod.rs | 2 +- multipart/src/client/sized.rs | 2 +- multipart/src/mock.rs | 8 +++++--- multipart/src/server/field.rs | 17 +++++++---------- multipart/src/server/mod.rs | 2 +- multipart/src/server/nickel.rs | 2 +- multipart/src/server/save.rs | 10 +++++----- 9 files changed, 27 insertions(+), 23 deletions(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 7516948cf..92d488a59 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -16,6 +16,8 @@ documentation = "http://docs.rs/multipart/" license = "MIT OR Apache-2.0" readme = "README.md" +autobins = false +edition = "2021" [dependencies] lazy_static = { version = "1.2.0", optional = true } @@ -28,7 +30,9 @@ tempfile = "3" clippy = { version = ">=0.0, <0.1", optional = true} #Server Dependencies + buf_redux = { version = "0.8", optional = true, default-features = false } + httparse = { version = "1.2", optional = true } twoway = { version = "0.1", optional = true } quick-error = { version = "1.2", optional = true } diff --git a/multipart/src/client/lazy.rs b/multipart/src/client/lazy.rs index 4df40cfe5..d19bb242a 100644 --- a/multipart/src/client/lazy.rs +++ b/multipart/src/client/lazy.rs @@ -486,6 +486,7 @@ fn cursor_at_end>(cursor: &Cursor) -> bool { mod hyper { use hyper::client::{Body, Client, IntoUrl, RequestBuilder, Response}; use hyper::Result as HyperResult; + use crate::client; impl<'n, 'd> super::Multipart<'n, 'd> { /// #### Feature: `hyper` @@ -522,7 +523,7 @@ mod hyper { }; mut_fn(client.post(url)) - .header(::client::hyper::content_type(fields.boundary())) + .header(client::hyper::content_type(fields.boundary())) .body(fields.to_body()) .send() } diff --git a/multipart/src/client/mod.rs b/multipart/src/client/mod.rs index 3158d2a7d..b1ba9391f 100644 --- a/multipart/src/client/mod.rs +++ b/multipart/src/client/mod.rs @@ -207,7 +207,7 @@ impl HttpStream for io::Sink { } fn gen_boundary() -> String { - ::random_alphanumeric(BOUNDARY_LEN) + crate::random_alphanumeric(BOUNDARY_LEN) } fn open_stream( diff --git a/multipart/src/client/sized.rs b/multipart/src/client/sized.rs index 75a3945c0..874ad2253 100644 --- a/multipart/src/client/sized.rs +++ b/multipart/src/client/sized.rs @@ -6,7 +6,7 @@ // copied, modified, or distributed except according to those terms. //! Sized/buffered wrapper around `HttpRequest`. -use client::{HttpRequest, HttpStream}; +use super::{HttpRequest, HttpStream}; use std::io; use std::io::prelude::*; diff --git a/multipart/src/mock.rs b/multipart/src/mock.rs index 2947661bd..c6f9d4ee8 100644 --- a/multipart/src/mock.rs +++ b/multipart/src/mock.rs @@ -11,6 +11,8 @@ use std::fmt; use rand::{self, Rng}; use rand::prelude::ThreadRng; +use crate::{client, server}; + /// A mock implementation of `client::HttpRequest` which can spawn an `HttpBuffer`. /// /// `client::HttpRequest` impl requires the `client` feature. @@ -21,7 +23,7 @@ pub struct ClientRequest { } #[cfg(feature = "client")] -impl ::client::HttpRequest for ClientRequest { +impl client::HttpRequest for ClientRequest { type Stream = HttpBuffer; type Error = io::Error; @@ -103,7 +105,7 @@ impl Write for HttpBuffer { } #[cfg(feature = "client")] -impl ::client::HttpStream for HttpBuffer { +impl client::HttpStream for HttpBuffer { type Request = ClientRequest; type Response = HttpBuffer; type Error = io::Error; @@ -165,7 +167,7 @@ impl<'a> Read for ServerRequest<'a> { } #[cfg(feature = "server")] -impl<'a> ::server::HttpRequest for ServerRequest<'a> { +impl<'a> server::HttpRequest for ServerRequest<'a> { type Body = Self; fn multipart_boundary(&self) -> Option<&str> { Some(self.boundary) } diff --git a/multipart/src/server/field.rs b/multipart/src/server/field.rs index 6ccf9525b..1a6849819 100644 --- a/multipart/src/server/field.rs +++ b/multipart/src/server/field.rs @@ -8,7 +8,6 @@ //! `multipart` field header parsing. use mime::Mime; -use std::error::Error; use std::io::{self, BufRead, Read}; use std::{fmt, str}; @@ -27,7 +26,7 @@ macro_rules! invalid_cont_disp { return Err(ParseHeaderError::InvalidContDisp( $reason, $cause.to_string(), - )); + )) }; } @@ -531,30 +530,28 @@ impl ReadEntryResult { } } -const GENERIC_PARSE_ERR: &str = "an error occurred while parsing field headers"; - quick_error! { #[derive(Debug)] enum ParseHeaderError { /// The `Content-Disposition` header was not found MissingContentDisposition(headers: String) { - display(x) -> ("{}:\n{}", x.description(), headers) + display(x) -> ("{}:\n{}", x, headers) description("\"Content-Disposition\" header not found in field headers") } InvalidContDisp(reason: &'static str, cause: String) { - display(x) -> ("{}: {}: {}", x.description(), reason, cause) + display(x) -> ("{}: {}: {}", x, reason, cause) description("invalid \"Content-Disposition\" header") } /// The header was found but could not be parsed TokenizeError(err: HttparseError) { - description(GENERIC_PARSE_ERR) - display(x) -> ("{}: {}", x.description(), err) + description("an error occurred while parsing field headers") + display(x) -> ("{}: {}", x, err) cause(err) from() } MimeError(cont_type: String) { description("Failed to parse Content-Type") - display(this) -> ("{}: {}", this.description(), cont_type) + display(this) -> ("{}: {}", this, cont_type) } TooLarge { description("field headers section ridiculously long or missing trailing CRLF-CRLF") @@ -562,7 +559,7 @@ quick_error! { /// IO error Io(err: io::Error) { description("an io error occurred while parsing the headers") - display(x) -> ("{}: {}", x.description(), err) + display(x) -> ("{}: {}", x, err) cause(err) from() } diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index 8cd01a0b8..7d8931d29 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -51,7 +51,7 @@ macro_rules! try_read_entry { ($self_:expr; $try:expr) => ( match $try { Ok(res) => res, - Err(err) => return ::server::ReadEntryResult::Error($self_, err), + Err(err) => return crate::server::ReadEntryResult::Error($self_, err), } ) } diff --git a/multipart/src/server/nickel.rs b/multipart/src/server/nickel.rs index 0f725a3b2..1cec1e1eb 100644 --- a/multipart/src/server/nickel.rs +++ b/multipart/src/server/nickel.rs @@ -7,7 +7,7 @@ use self::hyper::header::ContentType; pub use self::nickel::Request as NickelRequest; pub use self::nickel::hyper::server::Request as HyperRequest; -use server::{HttpRequest, Multipart}; +use crate::server::{HttpRequest, Multipart}; /// A wrapper for `&mut nickel::Request` which implements `multipart::server::HttpRequest`. /// diff --git a/multipart/src/server/save.rs b/multipart/src/server/save.rs index fa95c9ed8..372ff4384 100644 --- a/multipart/src/server/save.rs +++ b/multipart/src/server/save.rs @@ -6,9 +6,9 @@ // copied, modified, or distributed except according to those terms. //! Utilities for saving request entries to the filesystem. -pub use server::buf_redux::BufReader; +pub use crate::server::buf_redux::BufReader; -pub use tempfile::TempDir; +pub use crate::tempfile::TempDir; use std::collections::HashMap; use std::io::prelude::*; @@ -18,7 +18,7 @@ use std::sync::Arc; use std::{cmp, env, io, mem, str, u32, u64}; use tempfile; -use server::field::{FieldHeaders, MultipartField, MultipartData, ReadEntry, ReadEntryResult}; +use crate::server::field::{FieldHeaders, MultipartField, MultipartData, ReadEntry, ReadEntryResult}; use self::SaveResult::*; use self::TextPolicy::*; @@ -27,7 +27,7 @@ use self::PartialReason::*; const RANDOM_FILENAME_LEN: usize = 12; fn rand_filename() -> String { - ::random_alphanumeric(RANDOM_FILENAME_LEN) + crate::random_alphanumeric(RANDOM_FILENAME_LEN) } macro_rules! try_start ( @@ -664,7 +664,7 @@ impl Entries { Occupied(occupied) => { // dedup the field name by reusing the key's `Arc` headers.name = occupied.key().clone(); - occupied.into_mut().push({ SavedField { headers, data }}); + occupied.into_mut().push(SavedField { headers, data }); }, } From 83c16aa77fa5af0043652a69732e9ac0cc7f6f62 Mon Sep 17 00:00:00 2001 From: Trevor Gross Date: Sat, 14 Jun 2025 01:24:44 +0000 Subject: [PATCH 449/453] Replace `buf-redux` with `buffer-redux` `buffer-redux` [1] is a maintained fork of `buf-redux` [2]. Switch to it in `multipart`. [1]: https://docs.rs/buffer-redux/latest/buffer_redux/ [2]: https://docs.rs/buf_redux/latest/buf_redux/ --- multipart/Cargo.toml | 6 ++---- multipart/src/server/boundary.rs | 4 ++-- multipart/src/server/mod.rs | 2 +- multipart/src/server/save.rs | 2 +- 4 files changed, 6 insertions(+), 8 deletions(-) diff --git a/multipart/Cargo.toml b/multipart/Cargo.toml index 92d488a59..7f3dadbec 100644 --- a/multipart/Cargo.toml +++ b/multipart/Cargo.toml @@ -30,9 +30,7 @@ tempfile = "3" clippy = { version = ">=0.0, <0.1", optional = true} #Server Dependencies - -buf_redux = { version = "0.8", optional = true, default-features = false } - +buffer-redux = { version = "1.0", optional = true, default-features = false } httparse = { version = "1.2", optional = true } twoway = { version = "0.1", optional = true } quick-error = { version = "1.2", optional = true } @@ -52,7 +50,7 @@ env_logger = "0.5" [features] client = [] default = ["client", "hyper", "iron", "mock", "nickel", "server", "tiny_http"] -server = ["buf_redux", "httparse", "quick-error", "safemem", "twoway"] +server = ["buffer-redux", "httparse", "quick-error", "safemem", "twoway"] mock = [] nightly = [] bench = [] diff --git a/multipart/src/server/boundary.rs b/multipart/src/server/boundary.rs index 21f37d14d..37ccfda08 100644 --- a/multipart/src/server/boundary.rs +++ b/multipart/src/server/boundary.rs @@ -9,8 +9,8 @@ use ::safemem; -use super::buf_redux::BufReader; -use super::buf_redux::policy::MinBuffered; +use super::buffer_redux::BufReader; +use super::buffer_redux::policy::MinBuffered; use super::twoway; use std::cmp; diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index 7d8931d29..20fedc0ae 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -11,7 +11,7 @@ //! //! See the `Multipart` struct for more info. -pub extern crate buf_redux; +pub extern crate buffer_redux; extern crate httparse; extern crate twoway; diff --git a/multipart/src/server/save.rs b/multipart/src/server/save.rs index 372ff4384..7df9171cc 100644 --- a/multipart/src/server/save.rs +++ b/multipart/src/server/save.rs @@ -6,7 +6,7 @@ // copied, modified, or distributed except according to those terms. //! Utilities for saving request entries to the filesystem. -pub use crate::server::buf_redux::BufReader; +pub use crate::server::buffer_redux::BufReader; pub use crate::tempfile::TempDir; From 3efc2da6b6c849a80f50a76cadc8f953ba845e24 Mon Sep 17 00:00:00 2001 From: Trevor Gross Date: Sat, 14 Jun 2025 00:26:06 +0000 Subject: [PATCH 450/453] Switch to using the local `multipart` in Rouille --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index bd6103d7a..1c20eec10 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,7 +22,7 @@ brotli = { version = "3.3.2", optional = true } chrono = { version = "0.4.19", default-features = false, features = ["clock"] } filetime = "0.2.0" deflate = { version = "1.0.0", optional = true, features = ["gzip"] } -multipart = { version = "0.18", default-features = false, features = ["server"] } +multipart = { version = "0.18", path = "multipart", default-features = false, features = ["server"] } percent-encoding = "2" rand = "0.8" serde = "1" From 3aa39a19973d19809625eea0b2777f025b62c45d Mon Sep 17 00:00:00 2001 From: Trevor Gross Date: Sat, 14 Jun 2025 01:35:07 +0000 Subject: [PATCH 451/453] Add `multipart` to the workspace --- Cargo.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Cargo.toml b/Cargo.toml index 1c20eec10..2945ed388 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,3 +37,6 @@ sha1_smol = "1.0.0" [dev-dependencies] postgres = { version = "0.19", default-features = false } log = "0.4" + +[workspace] +members = ["multipart"] From 27514a62bd8cff97b27821db278cf6f12a5bb588 Mon Sep 17 00:00:00 2001 From: Trevor Gross Date: Sat, 14 Jun 2025 18:10:51 +0000 Subject: [PATCH 452/453] Run `cargo fmt` to update `multipart` --- multipart/examples/hyper_client.rs | 19 +-- multipart/examples/hyper_reqbuilder.rs | 2 +- multipart/examples/hyper_server.rs | 22 +-- multipart/examples/iron.rs | 25 +-- multipart/examples/iron_intercept.rs | 8 +- multipart/examples/nickel.rs | 36 ++-- multipart/examples/rocket.rs | 32 ++-- multipart/examples/tiny_http.rs | 25 +-- multipart/src/bin/form_test.rs | 13 +- multipart/src/client/hyper.rs | 19 ++- multipart/src/client/lazy.rs | 2 +- multipart/src/client/sized.rs | 31 ++-- multipart/src/mock.rs | 22 ++- multipart/src/server/boundary.rs | 117 ++++++++----- multipart/src/server/hyper.rs | 68 ++++---- multipart/src/server/iron.rs | 126 +++++++++----- multipart/src/server/mod.rs | 106 +++++++----- multipart/src/server/nickel.rs | 21 ++- multipart/src/server/save.rs | 224 +++++++++++++++++-------- 19 files changed, 575 insertions(+), 343 deletions(-) diff --git a/multipart/examples/hyper_client.rs b/multipart/examples/hyper_client.rs index 8e968f758..a8bdaad27 100644 --- a/multipart/examples/hyper_client.rs +++ b/multipart/examples/hyper_client.rs @@ -10,23 +10,21 @@ use multipart::client::Multipart; use std::io::Read; fn main() { - let url = "http://localhost:80".parse() - .expect("Failed to parse URL"); + let url = "http://localhost:80".parse().expect("Failed to parse URL"); - let request = Request::new(Method::Post, url) - .expect("Failed to create request"); + let request = Request::new(Method::Post, url).expect("Failed to create request"); - let mut multipart = Multipart::from_request(request) - .expect("Failed to create Multipart"); + let mut multipart = Multipart::from_request(request).expect("Failed to create Multipart"); - write_body(&mut multipart) - .expect("Failed to write multipart body"); + write_body(&mut multipart).expect("Failed to write multipart body"); let mut response = multipart.send().expect("Failed to send multipart request"); if !response.status.is_success() { let mut res = String::new(); - response.read_to_string(&mut res).expect("failed to read response"); + response + .read_to_string(&mut res) + .expect("failed to read response"); println!("response reported unsuccessful: {:?}\n {}", response, res); } @@ -39,6 +37,7 @@ fn write_body(multi: &mut Multipart>) -> hyper::Result<()> { multi.write_text("text", "Hello, world!")?; multi.write_file("file", "lorem_ipsum.txt")?; // &[u8] impl Read - multi.write_stream("binary", &mut binary, None, None) + multi + .write_stream("binary", &mut binary, None, None) .and(Ok(())) } diff --git a/multipart/examples/hyper_reqbuilder.rs b/multipart/examples/hyper_reqbuilder.rs index 05c5539c8..06c0bbc0e 100644 --- a/multipart/examples/hyper_reqbuilder.rs +++ b/multipart/examples/hyper_reqbuilder.rs @@ -16,4 +16,4 @@ fn main() { // Request is sent here .client_request(&Client::new(), "http://localhost:80") .expect("Error sending multipart request"); -} \ No newline at end of file +} diff --git a/multipart/examples/hyper_server.rs b/multipart/examples/hyper_server.rs index 56ebc8a88..30333323d 100644 --- a/multipart/examples/hyper_server.rs +++ b/multipart/examples/hyper_server.rs @@ -1,13 +1,13 @@ extern crate hyper; extern crate multipart; -use std::io; -use hyper::server::{Handler, Server, Request, Response}; -use hyper::status::StatusCode; use hyper::server::response::Response as HyperResponse; -use multipart::server::hyper::{Switch, MultipartHandler, HyperRequest}; -use multipart::server::{Multipart, Entries, SaveResult}; +use hyper::server::{Handler, Request, Response, Server}; +use hyper::status::StatusCode; use multipart::mock::StdoutTee; +use multipart::server::hyper::{HyperRequest, MultipartHandler, Switch}; +use multipart::server::{Entries, Multipart, SaveResult}; +use std::io; struct NonMultipart; impl Handler for NonMultipart { @@ -28,7 +28,8 @@ impl MultipartHandler for EchoMultipart { } SaveResult::Error(error) => { println!("Errors saving multipart:\n{:?}", error); - res.send(format!("An error occurred {}", error).as_bytes()).unwrap(); + res.send(format!("An error occurred {}", error).as_bytes()) + .unwrap(); } }; } @@ -43,9 +44,8 @@ fn process_entries(res: HyperResponse, entries: Entries) -> io::Result<()> { fn main() { println!("Listening on 0.0.0.0:3333"); - Server::http("0.0.0.0:3333").unwrap().handle( - Switch::new( - NonMultipart, - EchoMultipart - )).unwrap(); + Server::http("0.0.0.0:3333") + .unwrap() + .handle(Switch::new(NonMultipart, EchoMultipart)) + .unwrap(); } diff --git a/multipart/examples/iron.rs b/multipart/examples/iron.rs index effeef68e..944c0b34b 100644 --- a/multipart/examples/iron.rs +++ b/multipart/examples/iron.rs @@ -1,18 +1,20 @@ -extern crate multipart; extern crate iron; +extern crate multipart; extern crate env_logger; -use std::io::{self, Write}; -use multipart::mock::StdoutTee; -use multipart::server::{Multipart, Entries, SaveResult}; use iron::prelude::*; use iron::status; +use multipart::mock::StdoutTee; +use multipart::server::{Entries, Multipart, SaveResult}; +use std::io::{self, Write}; fn main() { env_logger::init(); - Iron::new(process_request).http("localhost:80").expect("Could not bind localhost:80"); + Iron::new(process_request) + .http("localhost:80") + .expect("Could not bind localhost:80"); } /// Processes a request and returns response or an occured error. @@ -29,18 +31,19 @@ fn process_request(request: &mut Request) -> IronResult { process_entries(entries.keep_partial())?; Ok(Response::with(( status::BadRequest, - format!("error reading request: {}", reason.unwrap_err()) + format!("error reading request: {}", reason.unwrap_err()), ))) } SaveResult::Error(error) => Ok(Response::with(( status::BadRequest, - format!("error reading request: {}", error) + format!("error reading request: {}", error), ))), } } - Err(_) => { - Ok(Response::with((status::BadRequest, "The request is not multipart"))) - } + Err(_) => Ok(Response::with(( + status::BadRequest, + "The request is not multipart", + ))), } } @@ -55,7 +58,7 @@ fn process_entries(entries: Entries) -> IronResult { entries.write_debug(tee).map_err(|e| { IronError::new( e, - (status::InternalServerError, "Error printing request fields") + (status::InternalServerError, "Error printing request fields"), ) })?; } diff --git a/multipart/examples/iron_intercept.rs b/multipart/examples/iron_intercept.rs index fa328dd6e..d9f5ba0d6 100644 --- a/multipart/examples/iron_intercept.rs +++ b/multipart/examples/iron_intercept.rs @@ -3,18 +3,18 @@ extern crate multipart; use iron::prelude::*; -use multipart::server::Entries; use multipart::server::iron::Intercept; +use multipart::server::Entries; fn main() { // We start with a basic request handler chain. - let mut chain = Chain::new(|req: &mut Request| + let mut chain = Chain::new(|req: &mut Request| { if let Some(entries) = req.extensions.get::() { Ok(Response::with(format!("{:?}", entries))) } else { Ok(Response::with("Not a multipart request")) } - ); + }); // `Intercept` will read out the entries and place them as an extension in the request. // It has various builder-style methods for changing how it will behave, but has sane settings @@ -22,4 +22,4 @@ fn main() { chain.link_before(Intercept::default()); Iron::new(chain).http("localhost:80").unwrap(); -} \ No newline at end of file +} diff --git a/multipart/examples/nickel.rs b/multipart/examples/nickel.rs index 71ec8ff3c..b1a5334fd 100644 --- a/multipart/examples/nickel.rs +++ b/multipart/examples/nickel.rs @@ -1,35 +1,33 @@ extern crate multipart; extern crate nickel; -use std::io::{self, Write}; -use nickel::{Action, HttpRouter, MiddlewareResult, Nickel, Request, Response}; use nickel::status::StatusCode; +use nickel::{Action, HttpRouter, MiddlewareResult, Nickel, Request, Response}; +use std::io::{self, Write}; +use multipart::mock::StdoutTee; use multipart::server::nickel::MultipartBody; use multipart::server::{Entries, SaveResult}; -use multipart::mock::StdoutTee; fn handle_multipart<'mw>(req: &mut Request, mut res: Response<'mw>) -> MiddlewareResult<'mw> { match (*req).multipart_body() { - Some(mut multipart) => { - match multipart.save().temp() { - SaveResult::Full(entries) => process_entries(res, entries), - - SaveResult::Partial(entries, e) => { - println!("Partial errors ... {:?}", e); - return process_entries(res, entries.keep_partial()); - }, - - SaveResult::Error(e) => { - println!("There are errors in multipart POSTing ... {:?}", e); - res.set(StatusCode::InternalServerError); - return res.send(format!("Server could not handle multipart POST! {:?}", e)); - }, + Some(mut multipart) => match multipart.save().temp() { + SaveResult::Full(entries) => process_entries(res, entries), + + SaveResult::Partial(entries, e) => { + println!("Partial errors ... {:?}", e); + return process_entries(res, entries.keep_partial()); } - } + + SaveResult::Error(e) => { + println!("There are errors in multipart POSTing ... {:?}", e); + res.set(StatusCode::InternalServerError); + return res.send(format!("Server could not handle multipart POST! {:?}", e)); + } + }, None => { res.set(StatusCode::BadRequest); - return res.send("Request seems not was a multipart request") + return res.send("Request seems not was a multipart request"); } } } diff --git a/multipart/examples/rocket.rs b/multipart/examples/rocket.rs index cf9e6f714..0cdb7953b 100644 --- a/multipart/examples/rocket.rs +++ b/multipart/examples/rocket.rs @@ -10,39 +10,45 @@ extern crate multipart; extern crate rocket; use multipart::mock::StdoutTee; -use multipart::server::Multipart; use multipart::server::save::Entries; use multipart::server::save::SaveResult::*; +use multipart::server::Multipart; -use rocket::Data; use rocket::http::{ContentType, Status}; -use rocket::response::Stream; use rocket::response::status::Custom; +use rocket::response::Stream; +use rocket::Data; use std::io::{self, Cursor, Write}; #[post("/upload", data = "")] // signature requires the request to have a `Content-Type` -fn multipart_upload(cont_type: &ContentType, data: Data) -> Result>>, Custom> { +fn multipart_upload( + cont_type: &ContentType, + data: Data, +) -> Result>>, Custom> { // this and the next check can be implemented as a request guard but it seems like just // more boilerplate than necessary if !cont_type.is_form_data() { return Err(Custom( Status::BadRequest, - "Content-Type not multipart/form-data".into() + "Content-Type not multipart/form-data".into(), )); } - let (_, boundary) = cont_type.params().find(|&(k, _)| k == "boundary").ok_or_else( - || Custom( + let (_, boundary) = cont_type + .params() + .find(|&(k, _)| k == "boundary") + .ok_or_else(|| { + Custom( Status::BadRequest, - "`Content-Type: multipart/form-data` boundary param not provided".into() + "`Content-Type: multipart/form-data` boundary param not provided".into(), ) - )?; + })?; match process_upload(boundary, data) { Ok(resp) => Ok(Stream::from(Cursor::new(resp))), - Err(err) => Err(Custom(Status::InternalServerError, err.to_string())) + Err(err) => Err(Custom(Status::InternalServerError, err.to_string())), } } @@ -61,7 +67,7 @@ fn process_upload(boundary: &str, data: Data) -> io::Result> { } process_entries(partial.entries, &mut out)? - }, + } Error(e) => return Err(e), } @@ -81,5 +87,7 @@ fn process_entries(entries: Entries, mut out: &mut Vec) -> io::Result<()> { } fn main() { - rocket::ignite().mount("/", routes![multipart_upload]).launch(); + rocket::ignite() + .mount("/", routes![multipart_upload]) + .launch(); } diff --git a/multipart/examples/tiny_http.rs b/multipart/examples/tiny_http.rs index caef10914..73b62abac 100644 --- a/multipart/examples/tiny_http.rs +++ b/multipart/examples/tiny_http.rs @@ -1,10 +1,10 @@ -extern crate tiny_http; extern crate multipart; +extern crate tiny_http; -use std::io::{self, Cursor, Write}; -use multipart::server::{Multipart, Entries, SaveResult}; use multipart::mock::StdoutTee; -use tiny_http::{Response, StatusCode, Request}; +use multipart::server::{Entries, Multipart, SaveResult}; +use std::io::{self, Cursor, Write}; +use tiny_http::{Request, Response, StatusCode}; fn main() { // Starting a server on `localhost:80` let server = tiny_http::Server::http("localhost:80").expect("Could not bind localhost:80"); @@ -18,7 +18,10 @@ fn main() { Ok(resp) => resp, Err(e) => { println!("An error has occured during request proccessing: {:?}", e); - build_response(500, "The received data was not correctly proccessed on the server") + build_response( + 500, + "The received data was not correctly proccessed on the server", + ) } }; @@ -70,9 +73,11 @@ fn process_entries(entries: Entries) -> io::Result> { fn build_response>>(status_code: u16, data: D) -> Response { let data = data.into(); let data_len = data.len(); - Response::new(StatusCode(status_code), - vec![], - Cursor::new(data), - Some(data_len), - None) + Response::new( + StatusCode(status_code), + vec![], + Cursor::new(data), + Some(data_len), + None, + ) } diff --git a/multipart/src/bin/form_test.rs b/multipart/src/bin/form_test.rs index 8cf1b9499..f039e8195 100644 --- a/multipart/src/bin/form_test.rs +++ b/multipart/src/bin/form_test.rs @@ -10,8 +10,10 @@ use std::fs::File; use std::io; fn main() { - let listening = Server::http("127.0.0.1:0").expect("failed to bind socket") - .handle(read_multipart).expect("failed to handle request"); + let listening = Server::http("127.0.0.1:0") + .expect("failed to bind socket") + .handle(read_multipart) + .expect("failed to handle request"); println!("bound socket to: {}", listening.socket); } @@ -23,10 +25,11 @@ fn read_multipart(req: Request, mut resp: Response) { } } - let mut file = File::open("src/bin/test_form.html") - .expect("failed to open src/bind/test_form.html"); + let mut file = + File::open("src/bin/test_form.html").expect("failed to open src/bind/test_form.html"); - resp.headers_mut().set(ContentType("text/html".parse().unwrap())); + resp.headers_mut() + .set(ContentType("text/html".parse().unwrap())); let mut resp = resp.start().expect("failed to open response"); io::copy(&mut file, &mut resp).expect("failed to write response"); diff --git a/multipart/src/client/hyper.rs b/multipart/src/client/hyper.rs index 675cba6d1..0d30be2ac 100644 --- a/multipart/src/client/hyper.rs +++ b/multipart/src/client/hyper.rs @@ -4,7 +4,7 @@ // http://apache.org/licenses/LICENSE-2.0> or the MIT license , at your option. This file may not be // copied, modified, or distributed except according to those terms. -//! Client-side integration with [Hyper](https://github.com/hyperium/hyper). +//! Client-side integration with [Hyper](https://github.com/hyperium/hyper). //! Enabled with the `hyper` feature (on by default). //! //! Contains `impl HttpRequest for Request` and `impl HttpStream for Request`. @@ -14,13 +14,13 @@ //! (adaptors for `hyper::client::RequestBuilder`). use hyper::client::request::Request; use hyper::client::response::Response; -use hyper::header::{ContentType, ContentLength}; +use hyper::header::{ContentLength, ContentType}; use hyper::method::Method; use hyper::net::{Fresh, Streaming}; use hyper::Error as HyperError; -use hyper::mime::{Mime, TopLevel, SubLevel, Attr, Value}; +use hyper::mime::{Attr, Mime, SubLevel, TopLevel, Value}; use super::{HttpRequest, HttpStream}; @@ -46,10 +46,10 @@ impl HttpRequest for Request { headers.set(ContentType(multipart_mime(boundary))); if let Some(size) = content_len { - headers.set(ContentLength(size)); + headers.set(ContentLength(size)); } - debug!("Hyper headers: {}", headers); + debug!("Hyper headers: {}", headers); true } @@ -57,7 +57,7 @@ impl HttpRequest for Request { fn open_stream(self) -> Result { self.start() } -} +} /// #### Feature: `hyper` impl HttpStream for Request { @@ -77,7 +77,8 @@ pub fn content_type(bound: &str) -> ContentType { fn multipart_mime(bound: &str) -> Mime { Mime( - TopLevel::Multipart, SubLevel::Ext("form-data".into()), - vec![(Attr::Ext("boundary".into()), Value::Ext(bound.into()))] - ) + TopLevel::Multipart, + SubLevel::Ext("form-data".into()), + vec![(Attr::Ext("boundary".into()), Value::Ext(bound.into()))], + ) } diff --git a/multipart/src/client/lazy.rs b/multipart/src/client/lazy.rs index d19bb242a..ca7cbb656 100644 --- a/multipart/src/client/lazy.rs +++ b/multipart/src/client/lazy.rs @@ -484,9 +484,9 @@ fn cursor_at_end>(cursor: &Cursor) -> bool { #[cfg(feature = "hyper")] mod hyper { + use crate::client; use hyper::client::{Body, Client, IntoUrl, RequestBuilder, Response}; use hyper::Result as HyperResult; - use crate::client; impl<'n, 'd> super::Multipart<'n, 'd> { /// #### Feature: `hyper` diff --git a/multipart/src/client/sized.rs b/multipart/src/client/sized.rs index 874ad2253..e13c2e5bc 100644 --- a/multipart/src/client/sized.rs +++ b/multipart/src/client/sized.rs @@ -14,16 +14,16 @@ use std::io::prelude::*; /// A wrapper around a request object that measures the request body and sets the `Content-Length` /// header to its size in bytes. /// -/// Sized requests are more human-readable and use less bandwidth +/// Sized requests are more human-readable and use less bandwidth /// (as chunking adds [visual noise and overhead][chunked-example]), /// but they must be able to load their entirety, including the contents of all files /// and streams, into memory so the request body can be measured. /// /// You should really only use sized requests if you intend to inspect the data manually on the /// server side, as it will produce a more human-readable request body. Also, of course, if the -/// server doesn't support chunked requests or otherwise rejects them. +/// server doesn't support chunked requests or otherwise rejects them. /// -/// [chunked-example]: http://en.wikipedia.org/wiki/Chunked_transfer_encoding#Example +/// [chunked-example]: http://en.wikipedia.org/wiki/Chunked_transfer_encoding#Example pub struct SizedRequest { inner: R, buffer: Vec, @@ -46,11 +46,15 @@ impl Write for SizedRequest { self.buffer.write(data) } - fn flush(&mut self) -> io::Result<()> { Ok(()) } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } } -impl HttpRequest for SizedRequest -where ::Error: From { +impl HttpRequest for SizedRequest +where + ::Error: From, +{ type Stream = Self; type Error = R::Error; @@ -67,20 +71,23 @@ where ::Error: From { } } -impl HttpStream for SizedRequest -where ::Error: From { +impl HttpStream for SizedRequest +where + ::Error: From, +{ type Request = Self; type Response = <::Stream as HttpStream>::Response; type Error = <::Stream as HttpStream>::Error; fn finish(mut self) -> Result { let content_len = self.buffer.len() as u64; - + if !self.inner.apply_headers(&self.boundary, Some(content_len)) { return Err(io::Error::new( - io::ErrorKind::Other, - "SizedRequest failed to apply headers to wrapped request." - ).into()); + io::ErrorKind::Other, + "SizedRequest failed to apply headers to wrapped request.", + ) + .into()); } let mut req = self.inner.open_stream()?; diff --git a/multipart/src/mock.rs b/multipart/src/mock.rs index c6f9d4ee8..1831cb4a9 100644 --- a/multipart/src/mock.rs +++ b/multipart/src/mock.rs @@ -5,11 +5,11 @@ // http://opensource.org/licenses/MIT>, at your option. This file may not be // copied, modified, or distributed except according to those terms. //! Mocked types for client-side and server-side APIs. -use std::io::{self, Read, Write}; use std::fmt; +use std::io::{self, Read, Write}; -use rand::{self, Rng}; use rand::prelude::ThreadRng; +use rand::{self, Rng}; use crate::{client, server}; @@ -37,13 +37,14 @@ impl client::HttpRequest for ClientRequest { /// If `apply_headers()` was not called. fn open_stream(self) -> Result { debug!("ClientRequest::open_stream called! {:?}", self); - let boundary = self.boundary.expect("ClientRequest::set_headers() was not called!"); + let boundary = self + .boundary + .expect("ClientRequest::set_headers() was not called!"); Ok(HttpBuffer::new_empty(boundary, self.content_len)) } } - /// A writable buffer which stores the boundary and content-length, if provided. /// /// Implements `client::HttpStream` if the `client` feature is enabled. @@ -69,7 +70,7 @@ impl HttpBuffer { buf, boundary, content_len, - rng: rand::thread_rng() + rng: rand::thread_rng(), } } @@ -111,7 +112,9 @@ impl client::HttpStream for HttpBuffer { type Error = io::Error; /// Returns `Ok(self)`. - fn finish(self) -> Result { Ok(self) } + fn finish(self) -> Result { + Ok(self) + } } impl fmt::Debug for HttpBuffer { @@ -170,7 +173,9 @@ impl<'a> Read for ServerRequest<'a> { impl<'a> server::HttpRequest for ServerRequest<'a> { type Body = Self; - fn multipart_boundary(&self) -> Option<&str> { Some(self.boundary) } + fn multipart_boundary(&self) -> Option<&str> { + Some(self.boundary) + } fn body(self) -> Self::Body { self @@ -187,7 +192,8 @@ impl<'s, W> StdoutTee<'s, W> { /// Constructor pub fn new(inner: W, stdout: &'s io::Stdout) -> Self { Self { - inner, stdout: stdout.lock(), + inner, + stdout: stdout.lock(), } } } diff --git a/multipart/src/server/boundary.rs b/multipart/src/server/boundary.rs index 37ccfda08..6d4fbf2f9 100644 --- a/multipart/src/server/boundary.rs +++ b/multipart/src/server/boundary.rs @@ -9,12 +9,12 @@ use ::safemem; -use super::buffer_redux::BufReader; use super::buffer_redux::policy::MinBuffered; +use super::buffer_redux::BufReader; use super::twoway; -use std::cmp; use std::borrow::Borrow; +use std::cmp; use std::io; use std::io::prelude::*; @@ -27,7 +27,7 @@ pub const MIN_BUF_SIZE: usize = 1024; enum State { Searching, BoundaryRead, - AtEnd + AtEnd, } /// A struct implementing `Read` and `BufRead` that will yield bytes until it sees a given sequence. @@ -39,7 +39,10 @@ pub struct BoundaryReader { state: State, } -impl BoundaryReader where R: Read { +impl BoundaryReader +where + R: Read, +{ /// Internal API pub fn from_reader>>(reader: R, boundary: B) -> BoundaryReader { let mut boundary = boundary.into(); @@ -59,11 +62,15 @@ impl BoundaryReader where R: Read { trace!("Buf: {:?}", String::from_utf8_lossy(buf)); - debug!("Before search Buf len: {} Search idx: {} State: {:?}", - buf.len(), self.search_idx, self.state); + debug!( + "Before search Buf len: {} Search idx: {} State: {:?}", + buf.len(), + self.search_idx, + self.state + ); if self.state == BoundaryRead || self.state == AtEnd { - return Ok(&buf[..self.search_idx]) + return Ok(&buf[..self.search_idx]); } if self.state == Searching && self.search_idx < buf.len() { @@ -74,22 +81,30 @@ impl BoundaryReader where R: Read { Ok(found_idx) => { self.search_idx += found_idx; self.state = BoundaryRead; - }, + } Err(yield_len) => { self.search_idx += yield_len; } } - } - - debug!("After search Buf len: {} Search idx: {} State: {:?}", - buf.len(), self.search_idx, self.state); + } + + debug!( + "After search Buf len: {} Search idx: {} State: {:?}", + buf.len(), + self.search_idx, + self.state + ); // back up the cursor to before the boundary's preceding CRLF if we haven't already if self.search_idx >= 2 && !buf[self.search_idx..].starts_with(b"\r\n") { - let two_bytes_before = &buf[self.search_idx - 2 .. self.search_idx]; + let two_bytes_before = &buf[self.search_idx - 2..self.search_idx]; - trace!("Two bytes before: {:?} ({:?}) (\"\\r\\n\": {:?})", - String::from_utf8_lossy(two_bytes_before), two_bytes_before, b"\r\n"); + trace!( + "Two bytes before: {:?} ({:?}) (\"\\r\\n\": {:?})", + String::from_utf8_lossy(two_bytes_before), + two_bytes_before, + b"\r\n" + ); if two_bytes_before == *b"\r\n" { debug!("Subtract two!"); @@ -122,8 +137,10 @@ impl BoundaryReader where R: Read { let buf_len = self.read_to_boundary()?.len(); if buf_len == 0 && self.state == Searching { - return Err(io::Error::new(io::ErrorKind::UnexpectedEof, - "unexpected end of request body")); + return Err(io::Error::new( + io::ErrorKind::UnexpectedEof, + "unexpected end of request body", + )); } debug!("Discarding {} bytes", buf_len); @@ -145,7 +162,7 @@ impl BoundaryReader where R: Read { consume_amt += 2; // assert that we've found the boundary after the CRLF - debug_assert_eq!(*self.boundary, bnd_segment[2 .. self.boundary.len() + 2]); + debug_assert_eq!(*self.boundary, bnd_segment[2..self.boundary.len() + 2]); } else { // assert that we've found the boundary debug_assert_eq!(*self.boundary, bnd_segment[..self.boundary.len()]); @@ -155,42 +172,51 @@ impl BoundaryReader where R: Read { consume_amt += 2; if buf.len() < consume_amt { - return Err(io::Error::new(io::ErrorKind::UnexpectedEof, - "not enough bytes to verify boundary")); + return Err(io::Error::new( + io::ErrorKind::UnexpectedEof, + "not enough bytes to verify boundary", + )); } // we have enough bytes to verify self.state = Searching; - let last_two = &buf[consume_amt - 2 .. consume_amt]; + let last_two = &buf[consume_amt - 2..consume_amt]; match last_two { b"\r\n" => self.state = Searching, b"--" => self.state = AtEnd, - _ => return Err(io::Error::new( - io::ErrorKind::InvalidData, - format!("unexpected bytes following multipart boundary: {:X} {:X}", - last_two[0], last_two[1]) - )), + _ => { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!( + "unexpected bytes following multipart boundary: {:X} {:X}", + last_two[0], last_two[1] + ), + )) + } } consume_amt }; - trace!("Consuming {} bytes, remaining buf: {:?}", - consume_amt, - String::from_utf8_lossy(self.source.buffer())); + trace!( + "Consuming {} bytes, remaining buf: {:?}", + consume_amt, + String::from_utf8_lossy(self.source.buffer()) + ); self.source.consume(consume_amt); - if cfg!(debug_assertions) { - - } + if cfg!(debug_assertions) {} self.search_idx = 0; - trace!("Consumed boundary (state: {:?}), remaining buf: {:?}", self.state, - String::from_utf8_lossy(self.source.buffer())); + trace!( + "Consumed boundary (state: {:?}), remaining buf: {:?}", + self.state, + String::from_utf8_lossy(self.source.buffer()) + ); Ok(self.state != AtEnd) } @@ -205,7 +231,7 @@ fn find_boundary(buf: &[u8], boundary: &[u8]) -> Result { let search_start = buf.len().saturating_sub(boundary.len()); // search for just the boundary fragment - for i in search_start .. buf.len() { + for i in search_start..buf.len() { if boundary.starts_with(&buf[i..]) { return Err(i); } @@ -234,7 +260,10 @@ impl Borrow for BoundaryReader { } } -impl Read for BoundaryReader where R: Read { +impl Read for BoundaryReader +where + R: Read, +{ fn read(&mut self, out: &mut [u8]) -> io::Result { let read = { let mut buf = self.read_to_boundary()?; @@ -247,7 +276,10 @@ impl Read for BoundaryReader where R: Read { } } -impl BufRead for BoundaryReader where R: Read { +impl BufRead for BoundaryReader +where + R: Read, +{ fn fill_buf(&mut self) -> io::Result<&[u8]> { self.read_to_boundary() } @@ -275,7 +307,7 @@ mod test { --boundary\r\n\ dashed-value-2\r\n\ --boundary--"; - + #[test] fn test_boundary() { ::init_log(); @@ -286,7 +318,7 @@ mod test { let mut reader = BoundaryReader::from_reader(src, BOUNDARY); let mut buf = String::new(); - + test_boundary_reader(&mut reader, &mut buf); } @@ -299,7 +331,7 @@ mod test { fn split(data: &'a [u8], at: usize) -> SplitReader<'a> { let (left, right) = data.split_at(at); - SplitReader { + SplitReader { left: left, right: right, } @@ -329,9 +361,9 @@ mod test { debug!("Testing boundary (split)"); let mut buf = String::new(); - + // Substitute for `.step_by()` being unstable. - for split_at in 0 .. TEST_VAL.len(){ + for split_at in 0..TEST_VAL.len() { debug!("Testing split at: {}", split_at); let src = SplitReader::split(TEST_VAL.as_bytes(), split_at); @@ -405,7 +437,6 @@ mod test { let ref mut buf = String::new(); let mut reader = BoundaryReader::from_reader(&mut body, BOUNDARY); - debug!("Consume 1"); assert_eq!(reader.consume_boundary().unwrap(), true); diff --git a/multipart/src/server/hyper.rs b/multipart/src/server/hyper.rs index 57022dc6d..2c9f5e2de 100644 --- a/multipart/src/server/hyper.rs +++ b/multipart/src/server/hyper.rs @@ -9,16 +9,16 @@ //! //! Also contains an implementation of [`HttpRequest`](../trait.HttpRequest.html) //! for `hyper::server::Request` and `&mut hyper::server::Request`. -use hyper::net::Fresh; use hyper::header::ContentType; use hyper::method::Method; +use hyper::net::Fresh; use hyper::server::{Handler, Request, Response}; pub use hyper::server::Request as HyperRequest; -use hyper::mime::{Mime, TopLevel, SubLevel, Attr, Value}; +use hyper::mime::{Attr, Mime, SubLevel, TopLevel, Value}; -use super::{Multipart, HttpRequest}; +use super::{HttpRequest, Multipart}; /// A container that implements `hyper::server::Handler` which will switch /// the handler implementation depending on if the incoming request is multipart or not. @@ -32,7 +32,11 @@ pub struct Switch { multipart: M, } -impl Switch where H: Handler, M: MultipartHandler { +impl Switch +where + H: Handler, + M: MultipartHandler, +{ /// Create a new `Switch` instance where /// `normal` handles normal Hyper requests and `multipart` handles Multipart requests pub fn new(normal: H, multipart: M) -> Switch { @@ -40,7 +44,11 @@ impl Switch where H: Handler, M: MultipartHandler { } } -impl Handler for Switch where H: Handler, M: MultipartHandler { +impl Handler for Switch +where + H: Handler, + M: MultipartHandler, +{ fn handle<'a, 'k>(&'a self, req: Request<'a, 'k>, res: Response<'a, Fresh>) { match Multipart::from_request(req) { Ok(multi) => self.multipart.handle_multipart(multi, res), @@ -55,16 +63,23 @@ impl Handler for Switch where H: Handler, M: MultipartHandler { /// and subsequently static functions. pub trait MultipartHandler: Send + Sync { /// Generate a response from this multipart request. - fn handle_multipart<'a, 'k>(&self, - multipart: Multipart>, - response: Response<'a, Fresh>); + fn handle_multipart<'a, 'k>( + &self, + multipart: Multipart>, + response: Response<'a, Fresh>, + ); } -impl MultipartHandler for F -where F: Fn(Multipart, Response), F: Send + Sync { - fn handle_multipart<'a, 'k>(&self, - multipart: Multipart>, - response: Response<'a, Fresh>) { +impl MultipartHandler for F +where + F: Fn(Multipart, Response), + F: Send + Sync, +{ + fn handle_multipart<'a, 'k>( + &self, + multipart: Multipart>, + response: Response<'a, Fresh>, + ) { (*self)(multipart, response); } } @@ -84,17 +99,16 @@ impl<'a, 'b> HttpRequest for HyperRequest<'a, 'b> { _ => return None, }; - params.iter().find(|&&(ref name, _)| - match *name { + params + .iter() + .find(|&&(ref name, _)| match *name { Attr::Boundary => true, _ => false, - } - ).and_then(|&(_, ref val)| - match *val { + }) + .and_then(|&(_, ref val)| match *val { Value::Ext(ref val) => Some(&**val), _ => None, - } - ) + }) }) } @@ -118,17 +132,16 @@ impl<'r, 'a, 'b> HttpRequest for &'r mut HyperRequest<'a, 'b> { _ => return None, }; - params.iter().find(|&&(ref name, _)| - match *name { + params + .iter() + .find(|&&(ref name, _)| match *name { Attr::Boundary => true, _ => false, - } - ).and_then(|&(_, ref val)| - match *val { + }) + .and_then(|&(_, ref val)| match *val { Value::Ext(ref val) => Some(&**val), _ => None, - } - ) + }) }) } @@ -136,4 +149,3 @@ impl<'r, 'a, 'b> HttpRequest for &'r mut HyperRequest<'a, 'b> { self } } - diff --git a/multipart/src/server/iron.rs b/multipart/src/server/iron.rs index c4bef5712..cd93157e0 100644 --- a/multipart/src/server/iron.rs +++ b/multipart/src/server/iron.rs @@ -4,7 +4,7 @@ //! iron::Request`. use iron::headers::ContentType; -use iron::mime::{Mime, TopLevel, SubLevel}; +use iron::mime::{Mime, SubLevel, TopLevel}; use iron::request::{Body as IronBody, Request as IronRequest}; use iron::typemap::Key; use iron::{BeforeMiddleware, IronError, IronResult}; @@ -13,9 +13,9 @@ use std::path::PathBuf; use std::{error, fmt, io}; use tempfile; -use super::{FieldHeaders, HttpRequest, Multipart}; -use super::save::{Entries, PartialReason, TempDir}; use super::save::SaveResult::*; +use super::save::{Entries, PartialReason, TempDir}; +use super::{FieldHeaders, HttpRequest, Multipart}; impl<'r, 'a, 'b> HttpRequest for &'r mut IronRequest<'a, 'b> { type Body = &'r mut IronBody<'a, 'b>; @@ -101,22 +101,34 @@ pub struct Intercept { impl Intercept { /// Set the `temp_dir_path` for this middleware. pub fn temp_dir_path>(self, path: P) -> Self { - Intercept { temp_dir_path: Some(path.into()), .. self } + Intercept { + temp_dir_path: Some(path.into()), + ..self + } } /// Set the `file_size_limit` for this middleware. pub fn file_size_limit(self, limit: u64) -> Self { - Intercept { file_size_limit: limit, .. self } + Intercept { + file_size_limit: limit, + ..self + } } /// Set the `file_count_limit` for this middleware. pub fn file_count_limit(self, limit: u32) -> Self { - Intercept { file_count_limit: limit, .. self } + Intercept { + file_count_limit: limit, + ..self + } } /// Set the `limit_behavior` for this middleware. pub fn limit_behavior(self, behavior: LimitBehavior) -> Self { - Intercept { limit_behavior: behavior, .. self } + Intercept { + limit_behavior: behavior, + ..self + } } fn read_request(&self, req: &mut IronRequest) -> IronResult> { @@ -125,12 +137,18 @@ impl Intercept { Err(_) => return Ok(None), }; - let tempdir = self.temp_dir_path.as_ref() - .map_or_else( - || tempfile::Builder::new().prefix("multipart-iron").tempdir(), - |path| tempfile::Builder::new().prefix("multipart-iron").tempdir_in(path) - ) - .map_err(|e| io_to_iron(e, "Error opening temporary directory for request."))?; + let tempdir = self + .temp_dir_path + .as_ref() + .map_or_else( + || tempfile::Builder::new().prefix("multipart-iron").tempdir(), + |path| { + tempfile::Builder::new() + .prefix("multipart-iron") + .tempdir_in(path) + }, + ) + .map_err(|e| io_to_iron(e, "Error opening temporary directory for request."))?; match self.limit_behavior { LimitBehavior::ThrowError => self.read_request_strict(multipart, tempdir), @@ -138,42 +156,66 @@ impl Intercept { } } - fn read_request_strict(&self, mut multipart: IronMultipart, tempdir: TempDir) -> IronResult> { - match multipart.save().size_limit(self.file_size_limit) - .count_limit(self.file_count_limit) - .with_temp_dir(tempdir) { + fn read_request_strict( + &self, + mut multipart: IronMultipart, + tempdir: TempDir, + ) -> IronResult> { + match multipart + .save() + .size_limit(self.file_size_limit) + .count_limit(self.file_count_limit) + .with_temp_dir(tempdir) + { Full(entries) => Ok(Some(entries)), Partial(_, PartialReason::Utf8Error(_)) => unreachable!(), - Partial(_, PartialReason::IoError(err)) => Err(io_to_iron(err, "Error midway through request")), - Partial(_, PartialReason::CountLimit) => Err(FileCountLimitError(self.file_count_limit).into()), - Partial(partial, PartialReason::SizeLimit) => { + Partial(_, PartialReason::IoError(err)) => { + Err(io_to_iron(err, "Error midway through request")) + } + Partial(_, PartialReason::CountLimit) => { + Err(FileCountLimitError(self.file_count_limit).into()) + } + Partial(partial, PartialReason::SizeLimit) => { let partial = partial.partial.expect(EXPECT_PARTIAL_FILE); - Err( - FileSizeLimitError { - field: partial.source.headers, - }.into() - ) - }, + Err(FileSizeLimitError { + field: partial.source.headers, + } + .into()) + } Error(err) => Err(io_to_iron(err, "Error at start of request")), } } - fn read_request_lenient(&self, mut multipart: IronMultipart, tempdir: TempDir) -> IronResult> { - let mut entries = match multipart.save().size_limit(self.file_size_limit) - .count_limit(self.file_count_limit) - .with_temp_dir(tempdir) { + fn read_request_lenient( + &self, + mut multipart: IronMultipart, + tempdir: TempDir, + ) -> IronResult> { + let mut entries = match multipart + .save() + .size_limit(self.file_size_limit) + .count_limit(self.file_count_limit) + .with_temp_dir(tempdir) + { Full(entries) => return Ok(Some(entries)), - Partial(_, PartialReason::IoError(err)) => return Err(io_to_iron(err, "Error midway through request")), - Partial(partial, _) => partial.keep_partial(), + Partial(_, PartialReason::IoError(err)) => { + return Err(io_to_iron(err, "Error midway through request")) + } + Partial(partial, _) => partial.keep_partial(), Error(err) => return Err(io_to_iron(err, "Error at start of request")), }; loop { - entries = match multipart.save().size_limit(self.file_size_limit) - .count_limit(self.file_count_limit) - .with_entries(entries) { + entries = match multipart + .save() + .size_limit(self.file_size_limit) + .count_limit(self.file_count_limit) + .with_entries(entries) + { Full(entries) => return Ok(Some(entries)), - Partial(_, PartialReason::IoError(err)) => return Err(io_to_iron(err, "Error midway through request")), + Partial(_, PartialReason::IoError(err)) => { + return Err(io_to_iron(err, "Error midway through request")) + } Partial(partial, _) => partial.keep_partial(), Error(err) => return Err(io_to_iron(err, "Error at start of request")), }; @@ -242,8 +284,16 @@ impl error::Error for FileSizeLimitError { impl fmt::Display for FileSizeLimitError { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match self.field.filename { - Some(ref filename) => write!(f, "File size limit reached for field \"{}\" (filename: \"{}\")", self.field.name, filename), - None => write!(f, "File size limit reached for field \"{}\" (no filename)", self.field.name), + Some(ref filename) => write!( + f, + "File size limit reached for field \"{}\" (filename: \"{}\")", + self.field.name, filename + ), + None => write!( + f, + "File size limit reached for field \"{}\" (no filename)", + self.field.name + ), } } } diff --git a/multipart/src/server/mod.rs b/multipart/src/server/mod.rs index 20fedc0ae..b0bd13d88 100644 --- a/multipart/src/server/mod.rs +++ b/multipart/src/server/mod.rs @@ -16,14 +16,14 @@ extern crate httparse; extern crate twoway; use std::borrow::Borrow; -use std::io::prelude::*; use std::io; +use std::io::prelude::*; use self::boundary::BoundaryReader; use self::field::PrivReadEntry; -pub use self::field::{FieldHeaders, MultipartField, MultipartData, ReadEntry, ReadEntryResult}; +pub use self::field::{FieldHeaders, MultipartData, MultipartField, ReadEntry, ReadEntryResult}; use self::save::SaveBuilder; @@ -48,12 +48,12 @@ macro_rules! try_opt ( ); macro_rules! try_read_entry { - ($self_:expr; $try:expr) => ( + ($self_:expr; $try:expr) => { match $try { Ok(res) => res, Err(err) => return crate::server::ReadEntryResult::Error($self_, err), } - ) + }; } mod boundary; @@ -91,8 +91,8 @@ impl Multipart<()> { None => return Err(req), }; - Ok(Multipart::with_body(req.body(), boundary)) - } + Ok(Multipart::with_body(req.body(), boundary)) + } } impl Multipart { @@ -110,7 +110,7 @@ impl Multipart { info!("Multipart::with_boundary(_, {:?})", boundary); - Multipart { + Multipart { reader: BoundaryReader::from_reader(body, boundary), } } @@ -132,12 +132,15 @@ impl Multipart { } /// Call `f` for each entry in the multipart request. - /// + /// /// This is a substitute for Rust not supporting streaming iterators (where the return value /// from `next()` borrows the iterator for a bound lifetime). /// /// Returns `Ok(())` when all fields have been read, or the first error. - pub fn foreach_entry(&mut self, mut foreach: F) -> io::Result<()> where F: FnMut(MultipartField<&mut Self>) { + pub fn foreach_entry(&mut self, mut foreach: F) -> io::Result<()> + where + F: FnMut(MultipartField<&mut Self>), + { loop { match self.read_entry() { Ok(Some(field)) => foreach(field), @@ -216,7 +219,9 @@ fn issue_104() { let request = Cursor::new(body); let mut multipart = Multipart::with_body(request, "boundary"); - multipart.foreach_entry(|_field| {/* Do nothing */}).unwrap_err(); + multipart + .foreach_entry(|_field| { /* Do nothing */ }) + .unwrap_err(); } #[test] @@ -226,7 +231,9 @@ fn issue_114() { fn consume_all(mut rdr: R) { loop { let consume = rdr.fill_buf().unwrap().len(); - if consume == 0 { return; } + if consume == 0 { + return; + } rdr.consume(consume); } } @@ -249,42 +256,51 @@ fn issue_114() { let mut multipart = Multipart::with_body(request, "------------------------c616e5fded96a3c7"); // one error if you do nothing - multipart.foreach_entry(|_entry| { /* do nothing */}).unwrap(); + multipart + .foreach_entry(|_entry| { /* do nothing */ }) + .unwrap(); // a different error if you skip the first field - multipart.foreach_entry(|entry| if *entry.headers.name != *"key1" { consume_all(entry.data); }) + multipart + .foreach_entry(|entry| { + if *entry.headers.name != *"key1" { + consume_all(entry.data); + } + }) .unwrap(); - - multipart.foreach_entry(|_entry| () /* match entry.headers.name.as_str() { - "file" => { - let mut vec = Vec::new(); - entry.data.read_to_end(&mut vec).expect("can't read"); - // message.file = String::from_utf8(vec).ok(); - println!("key file got"); - } - - "key1" => { - let mut vec = Vec::new(); - entry.data.read_to_end(&mut vec).expect("can't read"); - // message.key1 = String::from_utf8(vec).ok(); - println!("key1 got"); - } - - "key2" => { - let mut vec = Vec::new(); - entry.data.read_to_end(&mut vec).expect("can't read"); - // message.key2 = String::from_utf8(vec).ok(); - println!("key2 got"); - } - - _ => { - // as multipart has a bug https://github.com/abonander/multipart/issues/114 - // we manually do read_to_end here - //let mut _vec = Vec::new(); - //entry.data.read_to_end(&mut _vec).expect("can't read"); - println!("key neglected"); - } - }*/) - .expect("Unable to iterate multipart?") + multipart + .foreach_entry( + |_entry| (), /* match entry.headers.name.as_str() { + "file" => { + let mut vec = Vec::new(); + entry.data.read_to_end(&mut vec).expect("can't read"); + // message.file = String::from_utf8(vec).ok(); + println!("key file got"); + } + + "key1" => { + let mut vec = Vec::new(); + entry.data.read_to_end(&mut vec).expect("can't read"); + // message.key1 = String::from_utf8(vec).ok(); + println!("key1 got"); + } + + "key2" => { + let mut vec = Vec::new(); + entry.data.read_to_end(&mut vec).expect("can't read"); + // message.key2 = String::from_utf8(vec).ok(); + println!("key2 got"); + } + + _ => { + // as multipart has a bug https://github.com/abonander/multipart/issues/114 + // we manually do read_to_end here + //let mut _vec = Vec::new(); + //entry.data.read_to_end(&mut _vec).expect("can't read"); + println!("key neglected"); + } + }*/ + ) + .expect("Unable to iterate multipart?") } diff --git a/multipart/src/server/nickel.rs b/multipart/src/server/nickel.rs index 1cec1e1eb..1b4a603f8 100644 --- a/multipart/src/server/nickel.rs +++ b/multipart/src/server/nickel.rs @@ -1,11 +1,11 @@ //! Support for `multipart/form-data` bodies in [Nickel](https://nickel.rs). pub extern crate nickel; -use self::nickel::hyper; use self::hyper::header::ContentType; +use self::nickel::hyper; -pub use self::nickel::Request as NickelRequest; pub use self::nickel::hyper::server::Request as HyperRequest; +pub use self::nickel::Request as NickelRequest; use crate::server::{HttpRequest, Multipart}; @@ -43,27 +43,34 @@ impl<'mw, 'server, D: 'mw> MultipartBody<'mw, 'server> for NickelRequest<'mw, 's } } -impl<'r, 'mw: 'r, 'server: 'mw, D: 'mw> AsRef<&'r mut NickelRequest<'mw, 'server, D>> for Maybe<'r, 'mw, 'server, D> { +impl<'r, 'mw: 'r, 'server: 'mw, D: 'mw> AsRef<&'r mut NickelRequest<'mw, 'server, D>> + for Maybe<'r, 'mw, 'server, D> +{ fn as_ref(&self) -> &&'r mut NickelRequest<'mw, 'server, D> { &self.0 } } -impl<'r, 'mw: 'r, 'server: 'mw, D: 'mw> AsMut<&'r mut NickelRequest<'mw, 'server, D>> for Maybe<'r, 'mw, 'server, D> { +impl<'r, 'mw: 'r, 'server: 'mw, D: 'mw> AsMut<&'r mut NickelRequest<'mw, 'server, D>> + for Maybe<'r, 'mw, 'server, D> +{ fn as_mut(&mut self) -> &mut &'r mut NickelRequest<'mw, 'server, D> { &mut self.0 } } -impl<'r, 'mw: 'r, 'server: 'mw, D: 'mw> Into<&'r mut NickelRequest<'mw, 'server, D>> for Maybe<'r, 'mw, 'server, D> { +impl<'r, 'mw: 'r, 'server: 'mw, D: 'mw> Into<&'r mut NickelRequest<'mw, 'server, D>> + for Maybe<'r, 'mw, 'server, D> +{ fn into(self) -> &'r mut NickelRequest<'mw, 'server, D> { self.0 } } -impl<'r, 'mw: 'r, 'server: 'mw, D: 'mw> From<&'r mut NickelRequest<'mw, 'server, D>> for Maybe<'r, 'mw, 'server, D> { +impl<'r, 'mw: 'r, 'server: 'mw, D: 'mw> From<&'r mut NickelRequest<'mw, 'server, D>> + for Maybe<'r, 'mw, 'server, D> +{ fn from(req: &'r mut NickelRequest<'mw, 'server, D>) -> Self { Maybe(req) } } - diff --git a/multipart/src/server/save.rs b/multipart/src/server/save.rs index 7df9171cc..d39ee0993 100644 --- a/multipart/src/server/save.rs +++ b/multipart/src/server/save.rs @@ -11,18 +11,20 @@ pub use crate::server::buffer_redux::BufReader; pub use crate::tempfile::TempDir; use std::collections::HashMap; -use std::io::prelude::*; use std::fs::{self, File, OpenOptions}; +use std::io::prelude::*; use std::path::{Path, PathBuf}; use std::sync::Arc; use std::{cmp, env, io, mem, str, u32, u64}; use tempfile; -use crate::server::field::{FieldHeaders, MultipartField, MultipartData, ReadEntry, ReadEntryResult}; +use crate::server::field::{ + FieldHeaders, MultipartData, MultipartField, ReadEntry, ReadEntryResult, +}; +use self::PartialReason::*; use self::SaveResult::*; use self::TextPolicy::*; -use self::PartialReason::*; const RANDOM_FILENAME_LEN: usize = 12; @@ -65,7 +67,7 @@ enum TextPolicy { /// Attempt to read a text field as text, returning any errors Force, /// Don't try to read text - Ignore + Ignore, } /// A builder for saving a file or files to the local filesystem. @@ -185,7 +187,10 @@ impl SaveBuilder { /// If `0`, forces fields to save directly to the filesystem. /// If `u64::MAX`, effectively forces fields to always save to memory. pub fn memory_threshold(self, memory_threshold: u64) -> Self { - Self { memory_threshold, ..self } + Self { + memory_threshold, + ..self + } } /// When encountering a field that is apparently text, try to read it to a string or fall @@ -196,7 +201,10 @@ impl SaveBuilder { /// /// Has no effect once `memory_threshold` has been reached. pub fn try_text(self) -> Self { - Self { text_policy: TextPolicy::Try, ..self } + Self { + text_policy: TextPolicy::Try, + ..self + } } /// When encountering a field that is apparently text, read it to a string or return an error. @@ -206,17 +214,26 @@ impl SaveBuilder { /// /// (RFC: should this continue to validate UTF-8 when writing to the filesystem?) pub fn force_text(self) -> Self { - Self { text_policy: TextPolicy::Force, ..self} + Self { + text_policy: TextPolicy::Force, + ..self + } } /// Don't try to read or validate any field data as UTF-8. pub fn ignore_text(self) -> Self { - Self { text_policy: TextPolicy::Ignore, ..self } + Self { + text_policy: TextPolicy::Ignore, + ..self + } } } /// Save API for whole multipart requests. -impl SaveBuilder where M: ReadEntry { +impl SaveBuilder +where + M: ReadEntry, +{ /// Set the maximum number of fields to process. /// /// Can be `u32` or `Option`. If `None` or `u32::MAX`, clears the limit. @@ -287,8 +304,12 @@ impl SaveBuilder where M: ReadEntry { /// reaches `u32::MAX`, but this would be an extremely degenerate case. pub fn with_entries(self, mut entries: Entries) -> EntriesSaveResult { let SaveBuilder { - savable, open_opts, count_limit, size_limit, - memory_threshold, text_policy + savable, + open_opts, + count_limit, + size_limit, + memory_threshold, + text_policy, } = self; let mut res = ReadEntry::read_entry(savable); @@ -299,8 +320,12 @@ impl SaveBuilder where M: ReadEntry { let text_policy = if field.is_text() { text_policy } else { Ignore }; let mut saver = SaveBuilder { - savable: &mut field.data, open_opts: open_opts.clone(), - count_limit, size_limit, memory_threshold, text_policy + savable: &mut field.data, + open_opts: open_opts.clone(), + count_limit, + size_limit, + memory_threshold, + text_policy, }; saver.with_dir(entries.save_dir.as_path()) @@ -310,13 +335,15 @@ impl SaveBuilder where M: ReadEntry { let mut field: MultipartField = match res { ReadEntryResult::Entry(field) => field, ReadEntryResult::End(_) => return Full(entries), // normal exit point - ReadEntryResult::Error(_, e) => return Partial ( - PartialEntries { - entries, - partial: None, - }, - e.into(), - ) + ReadEntryResult::Error(_, e) => { + return Partial( + PartialEntries { + entries, + partial: None, + }, + e.into(), + ) + } }; let (dest, reason) = match save_field(&mut field, &entries) { @@ -324,7 +351,7 @@ impl SaveBuilder where M: ReadEntry { entries.push_field(field.headers, saved); res = ReadEntry::read_entry(field.data.into_inner()); continue; - }, + } Partial(saved, reason) => (Some(saved), reason), Error(error) => (None, PartialReason::IoError(error)), }; @@ -337,7 +364,7 @@ impl SaveBuilder where M: ReadEntry { dest, }), }, - reason + reason, ); } @@ -346,13 +373,16 @@ impl SaveBuilder where M: ReadEntry { entries, partial: None, }, - PartialReason::CountLimit + PartialReason::CountLimit, ) } } /// Save API for individual fields. -impl<'m, M: 'm> SaveBuilder<&'m mut MultipartData> where MultipartData: BufRead { +impl<'m, M: 'm> SaveBuilder<&'m mut MultipartData> +where + MultipartData: BufRead, +{ /// Save the field data, potentially using a file with a random name in the /// OS temporary directory. /// @@ -407,7 +437,7 @@ impl<'m, M: 'm> SaveBuilder<&'m mut MultipartData> where MultipartData: Bu match reason { SizeLimit if !self.cmp_size_limit(bytes.len()) => (), - other => return Partial(bytes.into(), other) + other => return Partial(bytes.into(), other), } let path = path.into(); @@ -417,15 +447,14 @@ impl<'m, M: 'm> SaveBuilder<&'m mut MultipartData> where MultipartData: Bu Err(e) => return Error(e), }; - let data = try_full!( - try_write_all(&bytes, &mut file) - .map(move |size| SavedData::File(path, size as u64)) - ); + let data = + try_full!(try_write_all(&bytes, &mut file) + .map(move |size| SavedData::File(path, size as u64))); - self.write_to(file).map(move |written| data.add_size(written)) + self.write_to(file) + .map(move |written| data.add_size(written)) } - /// Write out the field data to `dest`, truncating if a limit was set. /// /// Returns the number of bytes copied, and whether or not the limit was reached @@ -434,7 +463,11 @@ impl<'m, M: 'm> SaveBuilder<&'m mut MultipartData> where MultipartData: Bu /// Retries on interrupts. pub fn write_to(&mut self, mut dest: W) -> SaveResult { if self.size_limit < u64::MAX { - try_copy_limited(&mut self.savable, |buf| try_write_all(buf, &mut dest), self.size_limit) + try_copy_limited( + &mut self.savable, + |buf| try_write_all(buf, &mut dest), + self.size_limit, + ) } else { try_read_buf(&mut self.savable, |buf| try_write_all(buf, &mut dest)) } @@ -442,31 +475,49 @@ impl<'m, M: 'm> SaveBuilder<&'m mut MultipartData> where MultipartData: Bu fn save_mem(&mut self, mut bytes: Vec) -> SaveResult, Vec> { let pre_read = bytes.len() as u64; - match self.read_mem(|buf| { bytes.extend_from_slice(buf); Full(buf.len()) }, pre_read) { + match self.read_mem( + |buf| { + bytes.extend_from_slice(buf); + Full(buf.len()) + }, + pre_read, + ) { Full(_) => Full(bytes), Partial(_, reason) => Partial(bytes, reason), - Error(e) => if !bytes.is_empty() { Partial(bytes, e.into()) } - else { Error(e) } + Error(e) => { + if !bytes.is_empty() { + Partial(bytes, e.into()) + } else { + Error(e) + } + } } - } fn save_text(&mut self) -> SaveResult { let mut string = String::new(); // incrementally validate UTF-8 to do as much work as possible during network activity - let res = self.read_mem(|buf| { - match str::from_utf8(buf) { - Ok(s) => { string.push_str(s); Full(buf.len()) }, - // buffer should always be bigger - Err(e) => if buf.len() < 4 { - Partial(0, e.into()) - } else { - string.push_str(str::from_utf8(&buf[..e.valid_up_to()]).unwrap()); - Full(e.valid_up_to()) + let res = self.read_mem( + |buf| { + match str::from_utf8(buf) { + Ok(s) => { + string.push_str(s); + Full(buf.len()) } - } - }, 0); + // buffer should always be bigger + Err(e) => { + if buf.len() < 4 { + Partial(0, e.into()) + } else { + string.push_str(str::from_utf8(&buf[..e.valid_up_to()]).unwrap()); + Full(e.valid_up_to()) + } + } + } + }, + 0, + ); match res { Full(_) => Full(string), @@ -475,9 +526,12 @@ impl<'m, M: 'm> SaveBuilder<&'m mut MultipartData> where MultipartData: Bu } } - fn read_mem SaveResult>(&mut self, with_buf: Wb, pre_read: u64) -> SaveResult { - let limit = cmp::min(self.size_limit, self.memory_threshold) - .saturating_sub(pre_read); + fn read_mem SaveResult>( + &mut self, + with_buf: Wb, + pre_read: u64, + ) -> SaveResult { + let limit = cmp::min(self.size_limit, self.memory_threshold).saturating_sub(pre_read); try_copy_limited(&mut self.savable, with_buf, limit) } @@ -549,7 +603,7 @@ impl SavedData { match self { File(path, size) => File(path, size.saturating_add(add)), - other => other + other => other, } } } @@ -616,7 +670,7 @@ pub struct Entries { /// Each vector is guaranteed not to be empty unless externally modified. // Even though individual fields might only have one entry, it's better to limit the // size of a value type in `HashMap` to improve cache efficiency in lookups. - pub fields: HashMap, Vec>, + pub fields: HashMap, Vec>, /// The directory that the entries in `fields` were saved into. pub save_dir: SaveDir, fields_count: u32, @@ -660,12 +714,14 @@ impl Entries { use std::collections::hash_map::Entry::*; match self.fields.entry(headers.name.clone()) { - Vacant(vacant) => { vacant.insert(vec![SavedField { headers, data }]); }, + Vacant(vacant) => { + vacant.insert(vec![SavedField { headers, data }]); + } Occupied(occupied) => { // dedup the field name by reusing the key's `Arc` headers.name = occupied.key().clone(); occupied.into_mut().push(SavedField { headers, data }); - }, + } } self.fields_count = self.fields_count.saturating_add(1); @@ -686,7 +742,11 @@ impl Entries { for (idx, field) in entries.iter().enumerate() { let mut data = field.data.readable()?; let headers = &field.headers; - writeln!(writer, "{}: {:?} ({:?}):", idx, headers.filename, headers.content_type)?; + writeln!( + writer, + "{}: {:?} ({:?}):", + idx, headers.filename, headers.content_type + )?; io::copy(&mut data, &mut writer)?; } } @@ -907,7 +967,10 @@ impl EntriesSaveResult { } } -impl SaveResult where P: Into { +impl SaveResult +where + P: Into, +{ /// Convert `self` to `Option`; there may still have been an error. pub fn okish(self) -> Option { self.into_opt_both().0 @@ -915,7 +978,10 @@ impl SaveResult where P: Into { /// Map the `Full` or `Partial` values to a new type, retaining the reason /// in the `Partial` case. - pub fn map(self, map: Map) -> SaveResult where Map: FnOnce(S) -> T { + pub fn map(self, map: Map) -> SaveResult + where + Map: FnOnce(S) -> T, + { match self { Full(full) => Full(map(full)), Partial(partial, reason) => Partial(map(partial.into()), reason), @@ -926,7 +992,7 @@ impl SaveResult where P: Into { /// Decompose `self` to `(Option, Option)` pub fn into_opt_both(self) -> (Option, Option) { match self { - Full(full) => (Some(full), None), + Full(full) => (Some(full), None), Partial(partial, IoError(e)) => (Some(partial.into()), Some(e)), Partial(partial, _) => (Some(partial.into()), None), Error(error) => (None, Some(error)), @@ -964,23 +1030,35 @@ fn create_dir_all(path: &Path) -> io::Result<()> { fs::create_dir_all(parent) } else { // RFC: return an error instead? - warn!("Attempting to save file in what looks like a root directory. File path: {:?}", path); + warn!( + "Attempting to save file in what looks like a root directory. File path: {:?}", + path + ); Ok(()) } } -fn try_copy_limited SaveResult>(src: R, mut with_buf: Wb, limit: u64) -> SaveResult { +fn try_copy_limited SaveResult>( + src: R, + mut with_buf: Wb, + limit: u64, +) -> SaveResult { let mut copied = 0u64; try_read_buf(src, |buf| { let new_copied = copied.saturating_add(buf.len() as u64); - if new_copied > limit { return Partial(0, PartialReason::SizeLimit) } + if new_copied > limit { + return Partial(0, PartialReason::SizeLimit); + } copied = new_copied; with_buf(buf) }) } -fn try_read_buf SaveResult>(mut src: R, mut with_buf: Wb) -> SaveResult { +fn try_read_buf SaveResult>( + mut src: R, + mut with_buf: Wb, +) -> SaveResult { let mut total_copied = 0u64; macro_rules! try_here ( @@ -997,16 +1075,22 @@ fn try_read_buf SaveResult>(mut sr loop { let res = { let buf = try_here!(src.fill_buf()); - if buf.is_empty() { break; } + if buf.is_empty() { + break; + } with_buf(buf) }; match res { - Full(copied) => { src.consume(copied); total_copied += copied as u64; } + Full(copied) => { + src.consume(copied); + total_copied += copied as u64; + } Partial(copied, reason) => { - src.consume(copied); total_copied += copied as u64; + src.consume(copied); + total_copied += copied as u64; return Partial(total_copied, reason); - }, + } Error(err) => { return Partial(total_copied, err.into()); } @@ -1032,12 +1116,14 @@ fn try_write_all(mut buf: &[u8], mut dest: W) -> SaveResult try_here!(Err(io::Error::new(io::ErrorKind::WriteZero, - "failed to write whole buffer"))), + 0 => try_here!(Err(io::Error::new( + io::ErrorKind::WriteZero, + "failed to write whole buffer" + ))), copied => { buf = &buf[copied..]; total_copied += copied; - }, + } } } From ebb817cf899148c8d357e2dabdb3ae5fdf004f60 Mon Sep 17 00:00:00 2001 From: Trevor Gross Date: Sat, 14 Jun 2025 01:36:29 +0000 Subject: [PATCH 453/453] Rename `multipart` to `rouille-multipart` In preparation for republishing this crate as a version that suit Rouille's needs, rename it. --- Cargo.toml | 4 +- multipart/fuzz/Cargo.lock | 206 ------------------ {multipart => rouille-multipart}/.gitignore | 0 {multipart => rouille-multipart}/.travis.yml | 0 {multipart => rouille-multipart}/Cargo.toml | 10 +- {multipart => rouille-multipart}/LICENSE | 0 .../LICENSE-APACHE | 0 {multipart => rouille-multipart}/LICENSE-MIT | 0 {multipart => rouille-multipart}/README.md | 0 .../examples/README.md | 0 .../examples/hyper_client.rs | 0 .../examples/hyper_reqbuilder.rs | 0 .../examples/hyper_server.rs | 0 .../examples/iron.rs | 0 .../examples/iron_intercept.rs | 0 .../examples/nickel.rs | 0 .../examples/rocket.rs | 0 .../examples/tiny_http.rs | 0 .../fuzz/.gitignore | 0 .../fuzz/Cargo.toml | 0 .../fuzz/fuzzer_dict | 0 .../fuzz/fuzzers/logger.rs | 0 .../fuzz/fuzzers/server_basic.rs | 0 .../fuzz_server.sh | 0 .../lorem_ipsum.txt | 0 .../src/bin/form_test.rs | 0 .../src/bin/read_file.rs | 0 .../src/bin/test_form.html | 0 .../src/client/hyper.rs | 0 .../src/client/lazy.rs | 0 .../src/client/mod.rs | 0 .../src/client/sized.rs | 0 {multipart => rouille-multipart}/src/lib.rs | 0 .../src/local_test.rs | 0 {multipart => rouille-multipart}/src/mock.rs | 0 .../src/server/boundary.rs | 0 .../src/server/field.rs | 0 .../src/server/hyper.rs | 0 .../src/server/iron.rs | 0 .../src/server/mod.rs | 0 .../src/server/nickel.rs | 0 .../src/server/save.rs | 0 .../src/server/tiny_http.rs | 0 src/input/multipart.rs | 6 +- src/lib.rs | 2 +- 45 files changed, 7 insertions(+), 221 deletions(-) delete mode 100644 multipart/fuzz/Cargo.lock rename {multipart => rouille-multipart}/.gitignore (100%) rename {multipart => rouille-multipart}/.travis.yml (100%) rename {multipart => rouille-multipart}/Cargo.toml (98%) rename {multipart => rouille-multipart}/LICENSE (100%) rename {multipart => rouille-multipart}/LICENSE-APACHE (100%) rename {multipart => rouille-multipart}/LICENSE-MIT (100%) rename {multipart => rouille-multipart}/README.md (100%) rename {multipart => rouille-multipart}/examples/README.md (100%) rename {multipart => rouille-multipart}/examples/hyper_client.rs (100%) rename {multipart => rouille-multipart}/examples/hyper_reqbuilder.rs (100%) rename {multipart => rouille-multipart}/examples/hyper_server.rs (100%) rename {multipart => rouille-multipart}/examples/iron.rs (100%) rename {multipart => rouille-multipart}/examples/iron_intercept.rs (100%) rename {multipart => rouille-multipart}/examples/nickel.rs (100%) rename {multipart => rouille-multipart}/examples/rocket.rs (100%) rename {multipart => rouille-multipart}/examples/tiny_http.rs (100%) rename {multipart => rouille-multipart}/fuzz/.gitignore (100%) rename {multipart => rouille-multipart}/fuzz/Cargo.toml (100%) rename {multipart => rouille-multipart}/fuzz/fuzzer_dict (100%) rename {multipart => rouille-multipart}/fuzz/fuzzers/logger.rs (100%) rename {multipart => rouille-multipart}/fuzz/fuzzers/server_basic.rs (100%) rename {multipart => rouille-multipart}/fuzz_server.sh (100%) rename {multipart => rouille-multipart}/lorem_ipsum.txt (100%) rename {multipart => rouille-multipart}/src/bin/form_test.rs (100%) rename {multipart => rouille-multipart}/src/bin/read_file.rs (100%) rename {multipart => rouille-multipart}/src/bin/test_form.html (100%) rename {multipart => rouille-multipart}/src/client/hyper.rs (100%) rename {multipart => rouille-multipart}/src/client/lazy.rs (100%) rename {multipart => rouille-multipart}/src/client/mod.rs (100%) rename {multipart => rouille-multipart}/src/client/sized.rs (100%) rename {multipart => rouille-multipart}/src/lib.rs (100%) rename {multipart => rouille-multipart}/src/local_test.rs (100%) rename {multipart => rouille-multipart}/src/mock.rs (100%) rename {multipart => rouille-multipart}/src/server/boundary.rs (100%) rename {multipart => rouille-multipart}/src/server/field.rs (100%) rename {multipart => rouille-multipart}/src/server/hyper.rs (100%) rename {multipart => rouille-multipart}/src/server/iron.rs (100%) rename {multipart => rouille-multipart}/src/server/mod.rs (100%) rename {multipart => rouille-multipart}/src/server/nickel.rs (100%) rename {multipart => rouille-multipart}/src/server/save.rs (100%) rename {multipart => rouille-multipart}/src/server/tiny_http.rs (100%) diff --git a/Cargo.toml b/Cargo.toml index 2945ed388..13050dac0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,7 +22,7 @@ brotli = { version = "3.3.2", optional = true } chrono = { version = "0.4.19", default-features = false, features = ["clock"] } filetime = "0.2.0" deflate = { version = "1.0.0", optional = true, features = ["gzip"] } -multipart = { version = "0.18", path = "multipart", default-features = false, features = ["server"] } +rouille-multipart = { version = "0.18", path = "rouille-multipart", default-features = false, features = ["server"] } percent-encoding = "2" rand = "0.8" serde = "1" @@ -39,4 +39,4 @@ postgres = { version = "0.19", default-features = false } log = "0.4" [workspace] -members = ["multipart"] +members = ["rouille-multipart"] diff --git a/multipart/fuzz/Cargo.lock b/multipart/fuzz/Cargo.lock deleted file mode 100644 index 748229dd6..000000000 --- a/multipart/fuzz/Cargo.lock +++ /dev/null @@ -1,206 +0,0 @@ -[root] -name = "multipart-fuzz" -version = "0.0.1" -dependencies = [ - "libfuzzer-sys 0.1.0 (git+https://github.com/rust-fuzz/libfuzzer-sys.git)", - "log 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", - "multipart 0.11.0", -] - -[[package]] -name = "buf_redux" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "memchr 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)", - "safemem 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "gcc" -version = "0.3.45" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "httparse" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "libc" -version = "0.2.21" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "libfuzzer-sys" -version = "0.1.0" -source = "git+https://github.com/rust-fuzz/libfuzzer-sys.git#36a3928eef5c3c38eb0f251962395bb510c39d46" -dependencies = [ - "gcc 0.3.45 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "log" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "memchr" -version = "0.1.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "libc 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "mime" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "log 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "mime_guess" -version = "1.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "mime 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", - "phf 0.7.21 (registry+https://github.com/rust-lang/crates.io-index)", - "phf_codegen 0.7.21 (registry+https://github.com/rust-lang/crates.io-index)", - "unicase 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "multipart" -version = "0.11.0" -dependencies = [ - "buf_redux 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)", - "httparse 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", - "mime 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", - "mime_guess 1.8.1 (registry+https://github.com/rust-lang/crates.io-index)", - "rand 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)", - "safemem 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "tempdir 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", - "twoway 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "phf" -version = "0.7.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "phf_shared 0.7.21 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "phf_codegen" -version = "0.7.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "phf_generator 0.7.21 (registry+https://github.com/rust-lang/crates.io-index)", - "phf_shared 0.7.21 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "phf_generator" -version = "0.7.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "phf_shared 0.7.21 (registry+https://github.com/rust-lang/crates.io-index)", - "rand 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "phf_shared" -version = "0.7.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "siphasher 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "unicase 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "rand" -version = "0.3.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "libc 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "rustc_version" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "semver 0.1.20 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "safemem" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "safemem" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "semver" -version = "0.1.20" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "siphasher" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "tempdir" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "rand 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "twoway" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "memchr 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "unicase" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "rustc_version 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[metadata] -"checksum buf_redux 0.6.1 (registry+https://github.com/rust-lang/crates.io-index)" = "e1497634c131ba13483b6e8123f69e219253b018bb32949eefd55c6b5051585d" -"checksum gcc 0.3.45 (registry+https://github.com/rust-lang/crates.io-index)" = "40899336fb50db0c78710f53e87afc54d8c7266fb76262fecc78ca1a7f09deae" -"checksum httparse 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a6e7a63e511f9edffbab707141fbb8707d1a3098615fb2adbd5769cdfcc9b17d" -"checksum libc 0.2.21 (registry+https://github.com/rust-lang/crates.io-index)" = "88ee81885f9f04bff991e306fea7c1c60a5f0f9e409e99f6b40e3311a3363135" -"checksum libfuzzer-sys 0.1.0 (git+https://github.com/rust-fuzz/libfuzzer-sys.git)" = "" -"checksum log 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)" = "5141eca02775a762cc6cd564d8d2c50f67c0ea3a372cbf1c51592b3e029e10ad" -"checksum memchr 0.1.11 (registry+https://github.com/rust-lang/crates.io-index)" = "d8b629fb514376c675b98c1421e80b151d3817ac42d7c667717d282761418d20" -"checksum mime 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "5514f038123342d01ee5f95129e4ef1e0470c93bc29edf058a46f9ee3ba6737e" -"checksum mime_guess 1.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "76da6df85047af8c0edfa53f48eb1073012ce1cc95c8fedc0a374f659a89dd65" -"checksum phf 0.7.21 (registry+https://github.com/rust-lang/crates.io-index)" = "cb325642290f28ee14d8c6201159949a872f220c62af6e110a56ea914fbe42fc" -"checksum phf_codegen 0.7.21 (registry+https://github.com/rust-lang/crates.io-index)" = "d62594c0bb54c464f633175d502038177e90309daf2e0158be42ed5f023ce88f" -"checksum phf_generator 0.7.21 (registry+https://github.com/rust-lang/crates.io-index)" = "6b07ffcc532ccc85e3afc45865469bf5d9e4ef5bfcf9622e3cfe80c2d275ec03" -"checksum phf_shared 0.7.21 (registry+https://github.com/rust-lang/crates.io-index)" = "07e24b0ca9643bdecd0632f2b3da6b1b89bbb0030e0b992afc1113b23a7bc2f2" -"checksum rand 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)" = "022e0636ec2519ddae48154b028864bdce4eaf7d35226ab8e65c611be97b189d" -"checksum rustc_version 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "c5f5376ea5e30ce23c03eb77cbe4962b988deead10910c372b226388b594c084" -"checksum safemem 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "725b3bf47ae40b4abcd27b5f0a9540369426a29f7b905649b3e1468e13e22009" -"checksum safemem 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e27a8b19b835f7aea908818e871f5cc3a5a186550c30773be987e155e8163d8f" -"checksum semver 0.1.20 (registry+https://github.com/rust-lang/crates.io-index)" = "d4f410fedcf71af0345d7607d246e7ad15faaadd49d240ee3b24e5dc21a820ac" -"checksum siphasher 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "0df90a788073e8d0235a67e50441d47db7c8ad9debd91cbf43736a2a92d36537" -"checksum tempdir 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "87974a6f5c1dfb344d733055601650059a3363de2a6104819293baff662132d6" -"checksum twoway 0.1.3 (registry+https://github.com/rust-lang/crates.io-index)" = "e267e178055eb3b081224bbef62d4f508ae3c9f000b6ae6ccdb04a0d9c34b77f" -"checksum unicase 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "13a5906ca2b98c799f4b1ab4557b76367ebd6ae5ef14930ec841c74aed5f3764" diff --git a/multipart/.gitignore b/rouille-multipart/.gitignore similarity index 100% rename from multipart/.gitignore rename to rouille-multipart/.gitignore diff --git a/multipart/.travis.yml b/rouille-multipart/.travis.yml similarity index 100% rename from multipart/.travis.yml rename to rouille-multipart/.travis.yml diff --git a/multipart/Cargo.toml b/rouille-multipart/Cargo.toml similarity index 98% rename from multipart/Cargo.toml rename to rouille-multipart/Cargo.toml index 7f3dadbec..7511a801b 100644 --- a/multipart/Cargo.toml +++ b/rouille-multipart/Cargo.toml @@ -1,20 +1,12 @@ [package] -name = "multipart" - +name = "rouille-multipart" version = "0.18.0" - authors = ["Austin Bonander "] - description = "A backend-agnostic extension for HTTP libraries that provides support for POST multipart/form-data requests on both client and server." - keywords = ["form-data", "hyper", "iron", "http", "upload"] - repository = "http://github.com/abonander/multipart" - documentation = "http://docs.rs/multipart/" - license = "MIT OR Apache-2.0" - readme = "README.md" autobins = false edition = "2021" diff --git a/multipart/LICENSE b/rouille-multipart/LICENSE similarity index 100% rename from multipart/LICENSE rename to rouille-multipart/LICENSE diff --git a/multipart/LICENSE-APACHE b/rouille-multipart/LICENSE-APACHE similarity index 100% rename from multipart/LICENSE-APACHE rename to rouille-multipart/LICENSE-APACHE diff --git a/multipart/LICENSE-MIT b/rouille-multipart/LICENSE-MIT similarity index 100% rename from multipart/LICENSE-MIT rename to rouille-multipart/LICENSE-MIT diff --git a/multipart/README.md b/rouille-multipart/README.md similarity index 100% rename from multipart/README.md rename to rouille-multipart/README.md diff --git a/multipart/examples/README.md b/rouille-multipart/examples/README.md similarity index 100% rename from multipart/examples/README.md rename to rouille-multipart/examples/README.md diff --git a/multipart/examples/hyper_client.rs b/rouille-multipart/examples/hyper_client.rs similarity index 100% rename from multipart/examples/hyper_client.rs rename to rouille-multipart/examples/hyper_client.rs diff --git a/multipart/examples/hyper_reqbuilder.rs b/rouille-multipart/examples/hyper_reqbuilder.rs similarity index 100% rename from multipart/examples/hyper_reqbuilder.rs rename to rouille-multipart/examples/hyper_reqbuilder.rs diff --git a/multipart/examples/hyper_server.rs b/rouille-multipart/examples/hyper_server.rs similarity index 100% rename from multipart/examples/hyper_server.rs rename to rouille-multipart/examples/hyper_server.rs diff --git a/multipart/examples/iron.rs b/rouille-multipart/examples/iron.rs similarity index 100% rename from multipart/examples/iron.rs rename to rouille-multipart/examples/iron.rs diff --git a/multipart/examples/iron_intercept.rs b/rouille-multipart/examples/iron_intercept.rs similarity index 100% rename from multipart/examples/iron_intercept.rs rename to rouille-multipart/examples/iron_intercept.rs diff --git a/multipart/examples/nickel.rs b/rouille-multipart/examples/nickel.rs similarity index 100% rename from multipart/examples/nickel.rs rename to rouille-multipart/examples/nickel.rs diff --git a/multipart/examples/rocket.rs b/rouille-multipart/examples/rocket.rs similarity index 100% rename from multipart/examples/rocket.rs rename to rouille-multipart/examples/rocket.rs diff --git a/multipart/examples/tiny_http.rs b/rouille-multipart/examples/tiny_http.rs similarity index 100% rename from multipart/examples/tiny_http.rs rename to rouille-multipart/examples/tiny_http.rs diff --git a/multipart/fuzz/.gitignore b/rouille-multipart/fuzz/.gitignore similarity index 100% rename from multipart/fuzz/.gitignore rename to rouille-multipart/fuzz/.gitignore diff --git a/multipart/fuzz/Cargo.toml b/rouille-multipart/fuzz/Cargo.toml similarity index 100% rename from multipart/fuzz/Cargo.toml rename to rouille-multipart/fuzz/Cargo.toml diff --git a/multipart/fuzz/fuzzer_dict b/rouille-multipart/fuzz/fuzzer_dict similarity index 100% rename from multipart/fuzz/fuzzer_dict rename to rouille-multipart/fuzz/fuzzer_dict diff --git a/multipart/fuzz/fuzzers/logger.rs b/rouille-multipart/fuzz/fuzzers/logger.rs similarity index 100% rename from multipart/fuzz/fuzzers/logger.rs rename to rouille-multipart/fuzz/fuzzers/logger.rs diff --git a/multipart/fuzz/fuzzers/server_basic.rs b/rouille-multipart/fuzz/fuzzers/server_basic.rs similarity index 100% rename from multipart/fuzz/fuzzers/server_basic.rs rename to rouille-multipart/fuzz/fuzzers/server_basic.rs diff --git a/multipart/fuzz_server.sh b/rouille-multipart/fuzz_server.sh similarity index 100% rename from multipart/fuzz_server.sh rename to rouille-multipart/fuzz_server.sh diff --git a/multipart/lorem_ipsum.txt b/rouille-multipart/lorem_ipsum.txt similarity index 100% rename from multipart/lorem_ipsum.txt rename to rouille-multipart/lorem_ipsum.txt diff --git a/multipart/src/bin/form_test.rs b/rouille-multipart/src/bin/form_test.rs similarity index 100% rename from multipart/src/bin/form_test.rs rename to rouille-multipart/src/bin/form_test.rs diff --git a/multipart/src/bin/read_file.rs b/rouille-multipart/src/bin/read_file.rs similarity index 100% rename from multipart/src/bin/read_file.rs rename to rouille-multipart/src/bin/read_file.rs diff --git a/multipart/src/bin/test_form.html b/rouille-multipart/src/bin/test_form.html similarity index 100% rename from multipart/src/bin/test_form.html rename to rouille-multipart/src/bin/test_form.html diff --git a/multipart/src/client/hyper.rs b/rouille-multipart/src/client/hyper.rs similarity index 100% rename from multipart/src/client/hyper.rs rename to rouille-multipart/src/client/hyper.rs diff --git a/multipart/src/client/lazy.rs b/rouille-multipart/src/client/lazy.rs similarity index 100% rename from multipart/src/client/lazy.rs rename to rouille-multipart/src/client/lazy.rs diff --git a/multipart/src/client/mod.rs b/rouille-multipart/src/client/mod.rs similarity index 100% rename from multipart/src/client/mod.rs rename to rouille-multipart/src/client/mod.rs diff --git a/multipart/src/client/sized.rs b/rouille-multipart/src/client/sized.rs similarity index 100% rename from multipart/src/client/sized.rs rename to rouille-multipart/src/client/sized.rs diff --git a/multipart/src/lib.rs b/rouille-multipart/src/lib.rs similarity index 100% rename from multipart/src/lib.rs rename to rouille-multipart/src/lib.rs diff --git a/multipart/src/local_test.rs b/rouille-multipart/src/local_test.rs similarity index 100% rename from multipart/src/local_test.rs rename to rouille-multipart/src/local_test.rs diff --git a/multipart/src/mock.rs b/rouille-multipart/src/mock.rs similarity index 100% rename from multipart/src/mock.rs rename to rouille-multipart/src/mock.rs diff --git a/multipart/src/server/boundary.rs b/rouille-multipart/src/server/boundary.rs similarity index 100% rename from multipart/src/server/boundary.rs rename to rouille-multipart/src/server/boundary.rs diff --git a/multipart/src/server/field.rs b/rouille-multipart/src/server/field.rs similarity index 100% rename from multipart/src/server/field.rs rename to rouille-multipart/src/server/field.rs diff --git a/multipart/src/server/hyper.rs b/rouille-multipart/src/server/hyper.rs similarity index 100% rename from multipart/src/server/hyper.rs rename to rouille-multipart/src/server/hyper.rs diff --git a/multipart/src/server/iron.rs b/rouille-multipart/src/server/iron.rs similarity index 100% rename from multipart/src/server/iron.rs rename to rouille-multipart/src/server/iron.rs diff --git a/multipart/src/server/mod.rs b/rouille-multipart/src/server/mod.rs similarity index 100% rename from multipart/src/server/mod.rs rename to rouille-multipart/src/server/mod.rs diff --git a/multipart/src/server/nickel.rs b/rouille-multipart/src/server/nickel.rs similarity index 100% rename from multipart/src/server/nickel.rs rename to rouille-multipart/src/server/nickel.rs diff --git a/multipart/src/server/save.rs b/rouille-multipart/src/server/save.rs similarity index 100% rename from multipart/src/server/save.rs rename to rouille-multipart/src/server/save.rs diff --git a/multipart/src/server/tiny_http.rs b/rouille-multipart/src/server/tiny_http.rs similarity index 100% rename from multipart/src/server/tiny_http.rs rename to rouille-multipart/src/server/tiny_http.rs diff --git a/src/input/multipart.rs b/src/input/multipart.rs index b15bba188..da2b72ba2 100644 --- a/src/input/multipart.rs +++ b/src/input/multipart.rs @@ -18,11 +18,11 @@ use std::fmt; use Request; use RequestBody; -use multipart::server::Multipart as InnerMultipart; +use rouille_multipart::server::Multipart as InnerMultipart; // TODO: provide wrappers around these -pub use multipart::server::MultipartData; -pub use multipart::server::MultipartField; +pub use rouille_multipart::server::MultipartData; +pub use rouille_multipart::server::MultipartField; /// Error that can happen when decoding multipart data. #[derive(Clone, Debug)] diff --git a/src/lib.rs b/src/lib.rs index 37cf6cca6..2a6899804 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -63,7 +63,7 @@ extern crate chrono; #[cfg(feature = "gzip")] extern crate deflate; extern crate filetime; -extern crate multipart; +extern crate rouille_multipart; extern crate rand; extern crate serde; #[macro_use]