Filter by http status code

src/record.cc

@@ -86,7 +86,11 @@ namespace warc2text {
             // parse HTTP header
             pos = content.find("HTTP/1.", last_pos);
             if (pos == last_pos) { // found HTTP header
+                // parse HTTP status code
+                std::size_t space = content.find(" ", last_pos);
                 pos = content.find("\r\n", last_pos);
+                HTTPheader["status"] = content.substr(space + 1, (pos - space) - 1);
+
                 payload_start = read_header(content, pos + 2, HTTPheader);
                 if (payload_start == std::string::npos) {
                     // BOOST_LOG_TRIVIAL(warning) << "Response record without HTTP header";

src/warcpreprocessor.cc

@@ -78,7 +78,9 @@ namespace warc2text {
         totalBytes(0),
         textBytes(0),
         langBytes(0),
-        tagFilters() {
+        tagFilters(),
+        statusFilter("^20[036] ?.*$")
+    {
             if (!options.tag_filters_filename.empty())
                 util::readTagFiltersRegex(options.tag_filters_filename, tagFilters);
 
@@ -138,6 +140,10 @@ namespace warc2text {
             if (record.getRecordType() != "response" && record.getRecordType() != "resource")
                 continue;
 
+            if (record.HTTPheaderExists("status") && !boost::regex_match(record.getHTTPheaderProperty("status"), statusFilter)) {
+                continue;
+            }
+
             if (record.getWARCcontentType().find("application/http") == std::string::npos)
                 continue;
 

src/warcpreprocessor.hh

@@ -60,7 +60,8 @@ namespace warc2text {
             unsigned int langBytes;
             util::umap_tag_filters_regex tagFilters;
             boost::regex urlFilter;
-
+            const boost::regex statusFilter;
+
             static const std::unordered_set<std::string> removeExtensions;
             bool URLfilter(const std::string& url) const;