diff --git a/README b/README index 90bce81c..5172245d 100644 --- a/README +++ b/README @@ -5,31 +5,15 @@ Copyright 2010-2013 Qualys, Inc. ============================================================================ LibHTP is a security-aware parser for the HTTP protocol and the related bits -and pieces. The goals of the project, in the order of importance, are as -follows: +and pieces. The goal of the project is mainly to support the Suricata use case. +Other use cases might not fully be supported, and we encourage you to cover these. - 1. Completeness of coverage; LibHTP must be able to parse virtually all - traffic that is found in practice. + | STATUS + | + | We are currently in the process of migrating LibHTP to a Rust version and thus + | support will be discontinued. - 2. Permissive parsing; LibHTP must never fail to parse a stream that would - be parsed by some other web server. - - 3. Awareness of evasion techniques; LibHTP must be able to detect and - effectively deal with various evasion techniques, producing, where - practical, identical or practically identical results as the web - server processing the same traffic stream. - - 4. Performance; The performance must be adequate for the desired tasks. - Completeness and security are often detrimental to performance. Our - idea of handling the conflicting requirements is to put the library - user in control, allowing him to choose the most desired library - characteristic. - - | STATUS LIBHTP IS VERY YOUNG AT THIS POINT. IT WILL BE SOME TIME BEFORE - | IT CAN BE CONSIDER COMPLETE. AT THE MOMENT, THE FOCUS OF DEVELOPMENT - | IS ON ACHIEVING THE FIRST TWO GOALS. - -See the LICENSE, COPYING and NOTICE files distributed with this work for +See the LICENSE, COPYING, and NOTICE files distributed with this work for information regarding licensing, copying and copyright ownership. diff --git a/htp/htp_core.h b/htp/htp_core.h index d07f9913..c2eeb160 100644 --- a/htp/htp_core.h +++ b/htp/htp_core.h @@ -212,7 +212,10 @@ enum htp_content_encoding_t { HTP_COMPRESSION_DEFLATE = 3, /** LZMA compression. */ - HTP_COMPRESSION_LZMA = 4 + HTP_COMPRESSION_LZMA = 4, + + /** No more data. */ + HTP_COMPRESSION_OVER = 5 }; /** diff --git a/htp/htp_decompressors.c b/htp/htp_decompressors.c index 02ebae3e..d66b6e82 100644 --- a/htp/htp_decompressors.c +++ b/htp/htp_decompressors.c @@ -203,6 +203,8 @@ htp_status_t htp_gzip_decompressor_decompress(htp_decompressor_t *drec1, htp_tx_ } return HTP_OK; + } else if (drec->zlib_initialized == HTP_COMPRESSION_OVER) { + return HTP_ERROR; } if (d->data == NULL) { @@ -316,15 +318,8 @@ htp_status_t htp_gzip_decompressor_decompress(htp_decompressor_t *drec1, htp_tx_ // no initialization means previous error on stream return HTP_ERROR; } - if (GZIP_BUF_SIZE > drec->stream.avail_out) { - if (rc == Z_DATA_ERROR && drec->restart == 0) { - // There is data even if there is an error - // So use this data and log a warning - htp_log(d->tx->connp, HTP_LOG_MARK, HTP_LOG_WARNING, 0, "GZip decompressor: inflate failed with %d", rc); - rc = Z_STREAM_END; - } - } - if (rc == Z_STREAM_END) { + int error_after_data = (rc == Z_DATA_ERROR && drec->restart == 0 && GZIP_BUF_SIZE > drec->stream.avail_out); + if (rc == Z_STREAM_END || error_after_data) { // How many bytes do we have? size_t len = GZIP_BUF_SIZE - drec->stream.avail_out; @@ -351,6 +346,13 @@ htp_status_t htp_gzip_decompressor_decompress(htp_decompressor_t *drec1, htp_tx_ drec->stream.next_out = drec->buffer; // TODO Handle trailer. + if (error_after_data) { + // There is data even if there is an error + // So use this data and log a warning + htp_log(d->tx->connp, HTP_LOG_MARK, HTP_LOG_WARNING, 0, "GZip decompressor: inflate failed with %d", rc); + drec->zlib_initialized = HTP_COMPRESSION_OVER; + return HTP_ERROR; + } return HTP_OK; } else if (rc != Z_OK) { diff --git a/htp/htp_response.c b/htp/htp_response.c index abb4d590..7cef96ec 100644 --- a/htp/htp_response.c +++ b/htp/htp_response.c @@ -289,6 +289,12 @@ static void htp_connp_res_clear_buffer(htp_connp_t *connp) { htp_status_t htp_connp_RES_BODY_CHUNKED_DATA_END(htp_connp_t *connp) { // TODO We shouldn't really see anything apart from CR and LF, // so we should warn about anything else. + if (connp->out_status == HTP_STREAM_CLOSED) { + connp->out_state = htp_connp_RES_FINALIZE; + // Sends close signal to decompressors + htp_status_t rc = htp_tx_res_process_body_data_ex(connp->out_tx, NULL, 0); + return rc; + } for (;;) { OUT_NEXT_BYTE_OR_RETURN(connp); @@ -369,11 +375,6 @@ static inline int is_chunked_ctl_char(const unsigned char c) { * @returns 1 if it looks valid, 0 if it looks invalid */ static inline int data_probe_chunk_length(htp_connp_t *connp) { - if (connp->out_current_read_offset - connp->out_current_consume_offset < 8) { - // not enough data so far, consider valid still - return 1; - } - unsigned char *data = connp->out_current_data + connp->out_current_consume_offset; size_t len = connp->out_current_read_offset - connp->out_current_consume_offset; @@ -402,12 +403,19 @@ static inline int data_probe_chunk_length(htp_connp_t *connp) { * @returns HTP_OK on state change, HTP_ERROR on error, or HTP_DATA when more data is needed. */ htp_status_t htp_connp_RES_BODY_CHUNKED_LENGTH(htp_connp_t *connp) { + if (connp->out_status == HTP_STREAM_CLOSED) { + connp->out_state = htp_connp_RES_FINALIZE; + // Sends close signal to decompressors + htp_status_t rc = htp_tx_res_process_body_data_ex(connp->out_tx, NULL, 0); + return rc; + } + for (;;) { OUT_COPY_BYTE_OR_RETURN(connp); // Have we reached the end of the line? Or is this not chunked after all? if (connp->out_next_byte == LF || - (!is_chunked_ctl_char((unsigned char) connp->out_next_byte) && !data_probe_chunk_length(connp))) { + (!is_chunked_ctl_char((unsigned char) connp->out_next_byte) && !data_probe_chunk_length(connp) && connp->out_buf == NULL)) { unsigned char *data; size_t len; diff --git a/htp/htp_transaction.c b/htp/htp_transaction.c index 7220459d..9cde4155 100644 --- a/htp/htp_transaction.c +++ b/htp/htp_transaction.c @@ -972,8 +972,13 @@ htp_status_t htp_tx_res_process_body_data_ex(htp_tx_t *tx, const void *data, siz case HTP_COMPRESSION_DEFLATE: case HTP_COMPRESSION_LZMA: // In severe memory stress these could be NULL - if (tx->connp->out_decompressor == NULL) + if (tx->connp->out_decompressor == NULL) { + if (data == NULL) { + // we were already stopped on a gap finishing CL + return HTP_OK; + } return HTP_ERROR; + } struct timeval after; gettimeofday(&tx->connp->out_decompressor->time_before, NULL);