next/770/20250314/v1

README

@@ -5,31 +5,15 @@ Copyright 2010-2013 Qualys, Inc.
 ============================================================================
 
 LibHTP is a security-aware parser for the HTTP protocol and the related bits
-and pieces. The goals of the project, in the order of importance, are as
-follows:
+and pieces. The goal of the project is mainly to support the Suricata use case.
+Other use cases might not fully be supported, and we encourage you to cover these.
 
- 1. Completeness of coverage; LibHTP must be able to parse virtually all
-    traffic that is found in practice.
+   |   STATUS
+   |
+   |   We are currently in the process of migrating LibHTP to a Rust version and thus
+   |   support will be discontinued.
 
- 2. Permissive parsing; LibHTP must never fail to parse a stream that would
-    be parsed by some other web server.
-
- 3. Awareness of evasion techniques; LibHTP must be able to detect and
-    effectively deal with various evasion techniques, producing, where
-    practical, identical or practically identical results as the web
-    server processing the same traffic stream.
-
- 4. Performance; The performance must be adequate for the desired tasks.
-    Completeness and security are often detrimental to performance. Our
-    idea of handling the conflicting requirements is to put the library
-    user in control, allowing him to choose the most desired library
-    characteristic.
-
- | STATUS      LIBHTP IS VERY YOUNG AT THIS POINT. IT WILL BE SOME TIME BEFORE
- |             IT CAN BE CONSIDER COMPLETE. AT THE MOMENT, THE FOCUS OF DEVELOPMENT
- |             IS ON ACHIEVING THE FIRST TWO GOALS.
-
-See the LICENSE, COPYING and NOTICE files distributed with this work for
+See the LICENSE, COPYING, and NOTICE files distributed with this work for
 information regarding licensing, copying and copyright ownership.
 
 

htp/htp_core.h

@@ -212,7 +212,10 @@ enum htp_content_encoding_t {
     HTP_COMPRESSION_DEFLATE = 3,
 
     /** LZMA compression. */
-    HTP_COMPRESSION_LZMA = 4
+    HTP_COMPRESSION_LZMA = 4,
+
+    /** No more data. */
+    HTP_COMPRESSION_OVER = 5
 };
 
 /**

htp/htp_decompressors.c

@@ -203,6 +203,8 @@ htp_status_t htp_gzip_decompressor_decompress(htp_decompressor_t *drec1, htp_tx_
         }
 
         return HTP_OK;
+    } else if (drec->zlib_initialized == HTP_COMPRESSION_OVER) {
+        return HTP_ERROR;
     }
 
     if (d->data == NULL) {
@@ -316,15 +318,8 @@ htp_status_t htp_gzip_decompressor_decompress(htp_decompressor_t *drec1, htp_tx_
             // no initialization means previous error on stream
             return HTP_ERROR;
         }
-        if (GZIP_BUF_SIZE > drec->stream.avail_out) {
-            if (rc == Z_DATA_ERROR && drec->restart == 0) {
-                // There is data even if there is an error
-                // So use this data and log a warning
-                htp_log(d->tx->connp, HTP_LOG_MARK, HTP_LOG_WARNING, 0, "GZip decompressor: inflate failed with %d", rc);
-                rc = Z_STREAM_END;
-            }
-        }
-        if (rc == Z_STREAM_END) {
+        int error_after_data = (rc == Z_DATA_ERROR && drec->restart == 0 && GZIP_BUF_SIZE > drec->stream.avail_out);
+        if (rc == Z_STREAM_END || error_after_data) {
             // How many bytes do we have?
             size_t len = GZIP_BUF_SIZE - drec->stream.avail_out;
 
@@ -351,6 +346,13 @@ htp_status_t htp_gzip_decompressor_decompress(htp_decompressor_t *drec1, htp_tx_
             drec->stream.next_out = drec->buffer;
             // TODO Handle trailer.
 
+            if (error_after_data) {
+                // There is data even if there is an error
+                // So use this data and log a warning
+                htp_log(d->tx->connp, HTP_LOG_MARK, HTP_LOG_WARNING, 0, "GZip decompressor: inflate failed with %d", rc);
+                drec->zlib_initialized = HTP_COMPRESSION_OVER;
+                return HTP_ERROR;
+            }
             return HTP_OK;
         }
         else if (rc != Z_OK) {

htp/htp_response.c

@@ -289,6 +289,12 @@ static void htp_connp_res_clear_buffer(htp_connp_t *connp) {
 htp_status_t htp_connp_RES_BODY_CHUNKED_DATA_END(htp_connp_t *connp) {
     // TODO We shouldn't really see anything apart from CR and LF,
     //      so we should warn about anything else.
+    if (connp->out_status == HTP_STREAM_CLOSED) {
+        connp->out_state = htp_connp_RES_FINALIZE;
+        // Sends close signal to decompressors
+        htp_status_t rc = htp_tx_res_process_body_data_ex(connp->out_tx, NULL, 0);
+        return rc;
+    }
 
     for (;;) {
         OUT_NEXT_BYTE_OR_RETURN(connp);
@@ -369,11 +375,6 @@ static inline int is_chunked_ctl_char(const unsigned char c) {
  * @returns 1 if it looks valid, 0 if it looks invalid
  */
 static inline int data_probe_chunk_length(htp_connp_t *connp) {
-    if (connp->out_current_read_offset - connp->out_current_consume_offset < 8) {
-        // not enough data so far, consider valid still
-        return 1;
-    }
-
     unsigned char *data = connp->out_current_data + connp->out_current_consume_offset;
     size_t len = connp->out_current_read_offset - connp->out_current_consume_offset;
 
@@ -402,12 +403,19 @@ static inline int data_probe_chunk_length(htp_connp_t *connp) {
  * @returns HTP_OK on state change, HTP_ERROR on error, or HTP_DATA when more data is needed.
  */
 htp_status_t htp_connp_RES_BODY_CHUNKED_LENGTH(htp_connp_t *connp) {
+    if (connp->out_status == HTP_STREAM_CLOSED) {
+        connp->out_state = htp_connp_RES_FINALIZE;
+        // Sends close signal to decompressors
+        htp_status_t rc = htp_tx_res_process_body_data_ex(connp->out_tx, NULL, 0);
+        return rc;
+    }
+
     for (;;) {
         OUT_COPY_BYTE_OR_RETURN(connp);
 
         // Have we reached the end of the line? Or is this not chunked after all?
         if (connp->out_next_byte == LF ||
-                (!is_chunked_ctl_char((unsigned char) connp->out_next_byte) && !data_probe_chunk_length(connp))) {
+                (!is_chunked_ctl_char((unsigned char) connp->out_next_byte) && !data_probe_chunk_length(connp) && connp->out_buf == NULL)) {
             unsigned char *data;
             size_t len;
 

htp/htp_transaction.c

@@ -972,8 +972,13 @@ htp_status_t htp_tx_res_process_body_data_ex(htp_tx_t *tx, const void *data, siz
         case HTP_COMPRESSION_DEFLATE:
         case HTP_COMPRESSION_LZMA:
             // In severe memory stress these could be NULL
-            if (tx->connp->out_decompressor == NULL)
+            if (tx->connp->out_decompressor == NULL) {
+                if (data == NULL) {
+                    // we were already stopped on a gap finishing CL
+                    return HTP_OK;
+                }
                 return HTP_ERROR;
+            }
 
             struct timeval after;
             gettimeofday(&tx->connp->out_decompressor->time_before, NULL);