Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

TLS refactor #227

Merged
merged 6 commits into from
Feb 27, 2025
Merged

TLS refactor #227

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
d546286
Refactor TLS parser
Zadamsa Nov 3, 2024
e6604c8
Update IPFIX element for JA4
Zadamsa Nov 23, 2024
78aee80
Refactor TLS plugin
Zadamsa Nov 3, 2024
a0b756f
Add sha-256 hash
Zadamsa Nov 4, 2024
04192c1
Refactor QUIC Parser to use refactored TLS Parser
Zadamsa Nov 17, 2024
acbdca0
Update tests
Zadamsa Nov 23, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Makefile.am
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,7 @@ ipfixprobe_process_src=\
process/tls.hpp \
process/tls_parser.cpp \
process/tls_parser.hpp \
process/sha256.hpp \
process/smtp.cpp \
process/smtp.hpp \
process/dns-utils.hpp \
Expand Down
3 changes: 2 additions & 1 deletion include/ipfixprobe/ipfix-elements.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -185,10 +185,10 @@ namespace ipxp {
#define TLS_VERSION(F) F(39499, 333, 2, nullptr)
#define TLS_ALPN(F) F(39499, 337, -1, nullptr)
#define TLS_JA3(F) F(39499, 357, -1, nullptr)
#define TLS_JA4(F) F(39499, 358, -1, nullptr)
#define TLS_EXT_TYPE(F) F(0, 291, -1, nullptr)
#define TLS_EXT_LEN(F) F(0, 291, -1, nullptr)


#define SMTP_COMMANDS(F) F(8057, 810, 4, nullptr)
#define SMTP_MAIL_COUNT(F) F(8057, 811, 4, nullptr)
#define SMTP_RCPT_COUNT(F) F(8057, 812, 4, nullptr)
Expand Down Expand Up @@ -389,6 +389,7 @@ namespace ipxp {
F(TLS_SNI) \
F(TLS_ALPN) \
F(TLS_JA3) \
F(TLS_JA4) \
F(TLS_EXT_TYPE) \
F(TLS_EXT_LEN)

Expand Down
136 changes: 54 additions & 82 deletions process/quic_parser.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -248,95 +248,65 @@ uint64_t QUICParser::quic_get_variable_length(const uint8_t* start, uint64_t& of
}
} // QUICParser::quic_get_variable_length

bool QUICParser::quic_obtain_tls_data(TLSData& payload)
bool QUICParser::quic_parse_tls_extensions()
{
quic_tls_extension_lengths_pos = 0;
quic_tls_ext_type_pos = 0;
quic_tls_ext_pos = 0;
while (payload.start + sizeof(tls_ext) <= payload.end) {
tls_ext* ext = (tls_ext*) payload.start;
uint16_t type = ntohs(ext->type);
uint16_t length = ntohs(ext->length);

// Store extension type
if (quic_tls_ext_type_pos < MAX_QUIC_TLS_EXT_LEN) {
quic_tls_ext_type[quic_tls_ext_type_pos] = type;
quic_tls_ext_type_pos += 1;
}

// Store extension type length
if (quic_tls_extension_lengths_pos < MAX_QUIC_TLS_EXT_LEN) {
quic_tls_extension_lengths[quic_tls_extension_lengths_pos] = length;
quic_tls_extension_lengths_pos += 1;
}

//
payload.start += sizeof(tls_ext);

if (payload.start + length > payload.end) {
break;
}

// Save value payload except for length
if (quic_tls_ext_pos + length < CURRENT_BUFFER_SIZE) {
const bool extensions_parsed = tls_parser.parse_extensions([this](
uint16_t extension_type,
const uint8_t* extension_payload,
uint16_t extension_length) {
if (extension_type == TLS_EXT_SERVER_NAME && extension_length != 0) {
tls_parser.parse_server_names(extension_payload, extension_length);
} else if (
(extension_type == TLS_EXT_QUIC_TRANSPORT_PARAMETERS_V1
|| extension_type == TLS_EXT_QUIC_TRANSPORT_PARAMETERS
|| extension_type == TLS_EXT_QUIC_TRANSPORT_PARAMETERS_V2)
&& extension_length != 0) {
tls_parser.parse_quic_user_agent(extension_payload, extension_length);
}
if (quic_tls_ext_pos + extension_length < CURRENT_BUFFER_SIZE) {
#ifndef QUIC_CH_FULL_TLS_EXT
if (type == TLS_EXT_ALPN || type == TLS_EXT_QUIC_TRANSPORT_PARAMETERS_V1
|| type == TLS_EXT_QUIC_TRANSPORT_PARAMETERS
|| type == TLS_EXT_QUIC_TRANSPORT_PARAMETERS_V2) {
if (extension_type == TLS_EXT_ALPN || extension_type == TLS_EXT_QUIC_TRANSPORT_PARAMETERS_V1
|| extension_type == TLS_EXT_QUIC_TRANSPORT_PARAMETERS
|| extension_type == TLS_EXT_QUIC_TRANSPORT_PARAMETERS_V2) {
#endif
memcpy(quic_tls_ext + quic_tls_ext_pos, payload.start, length);
quic_tls_ext_pos += length;
memcpy(quic_tls_ext + quic_tls_ext_pos, extension_payload, extension_length);
quic_tls_ext_pos += extension_length;
#ifndef QUIC_CH_FULL_TLS_EXT
}
}
#endif
}

// Legacy extract specific fields
if (type == TLS_EXT_SERVER_NAME && length != 0) {
tls_parser.tls_get_server_name(payload, sni, BUFF_SIZE);
} else if (
(type == TLS_EXT_QUIC_TRANSPORT_PARAMETERS_V1
|| type == TLS_EXT_QUIC_TRANSPORT_PARAMETERS
|| type == TLS_EXT_QUIC_TRANSPORT_PARAMETERS_V2)
&& length != 0) {
tls_parser.tls_get_quic_user_agent(payload, user_agent, BUFF_SIZE);
}
payload.start += length;
}
return payload.obejcts_parsed != 0;
}
tls_parser.add_extension(extension_type, extension_length);
});
if (!extensions_parsed) {
return false;
}
tls_parser.save_server_names(sni, BUFF_SIZE);
tls_parser.save_quic_user_agent(user_agent, BUFF_SIZE);

const size_t copy_count = std::min<size_t>(tls_parser.get_extensions().size(), MAX_QUIC_TLS_EXT_LEN);
std::transform(tls_parser.get_extensions().begin(),
tls_parser.get_extensions().begin() + static_cast<ssize_t>(copy_count),
std::begin(quic_tls_ext_type),
[](const TLSExtension& typeLength) {
return typeLength.type;
});
std::transform(tls_parser.get_extensions().begin(),
tls_parser.get_extensions().begin() + static_cast<ssize_t>(copy_count),
std::begin(quic_tls_extension_lengths),
[](const TLSExtension& typeLength) {
return typeLength.length;
});
quic_tls_ext_type_pos = quic_tls_extension_lengths_pos = copy_count;
return true;
}

bool QUICParser::quic_parse_tls()
{
TLSData payload = {
payload.start = final_payload + quic_crypto_start,
payload.end = final_payload + quic_crypto_start + quic_crypto_len,
payload.obejcts_parsed = 0,
};

if (!tls_parser.tls_check_handshake(payload)) {
return false;
}
if (!tls_parser.tls_skip_random(payload)) {
return false;
}
if (!tls_parser.tls_skip_sessid(payload)) {
return false;
}
if (!tls_parser.tls_skip_cipher_suites(payload)) {
if (!tls_parser.parse_quic_tls(final_payload + quic_crypto_start, quic_crypto_len)) {
return false;
}
if (!tls_parser.tls_skip_compression_met(payload)) {
return false;
}
if (!tls_parser.tls_check_ext_len(payload)) {
return false;
}
// If no parameters were extracted. We also accept the QUIC connection. (no error check here)
quic_obtain_tls_data(payload);

return true;
} // QUICPlugin::quic_parse_tls
return quic_parse_tls_extensions();
}

uint8_t QUICParser::quic_draft_version(uint32_t version)
{
Expand Down Expand Up @@ -1394,14 +1364,16 @@ bool QUICParser::quic_parse_headers(const Packet& pkt, bool forceInitialParsing)

bool QUICParser::quic_set_server_port(const Packet& pkt)
{
tls_handshake hs = tls_parser.tls_get_handshake();
if (!tls_parser.get_handshake().has_value()) {
return false;
}

switch (packet_type) {
case INITIAL:
tls_hs_type = hs.type;
if (hs.type == 1) {
tls_hs_type = tls_parser.get_handshake()->type;
if (tls_hs_type == 1) {
server_port = pkt.dst_port;
} else if (hs.type == 2) {
} else if (tls_hs_type == 2) {
// Won't be reached, since we don't supply the OCCID to quic_parser
server_port = pkt.src_port;
}
Expand Down
2 changes: 1 addition & 1 deletion process/quic_parser.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,7 @@ class QUICParser {
uint64_t quic_get_variable_length(const uint8_t*, uint64_t&);
bool quic_check_version(uint32_t, uint8_t);
bool quic_check_pointer_pos(const uint8_t*, const uint8_t*);
bool quic_obtain_tls_data(TLSData&);
bool quic_parse_tls_extensions();
bool quic_set_server_port(const Packet& pkt);
bool quic_check_min_initial_size(const Packet& pkt);
bool quic_check_supported_version(const uint32_t version);
Expand Down
148 changes: 148 additions & 0 deletions process/sha256.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,148 @@
// Single file sha256
// https://github.com/LekKit/sha256

#include <stdio.h>
#include <string.h>

namespace sha256 {

struct sha256_buff {
unsigned long data_size;
unsigned int h[8];
unsigned char last_chunk[64];
unsigned char chunk_size;
};

void sha256_init(struct sha256_buff* buff)
{
buff->h[0] = 0x6a09e667;
buff->h[1] = 0xbb67ae85;
buff->h[2] = 0x3c6ef372;
buff->h[3] = 0xa54ff53a;
buff->h[4] = 0x510e527f;
buff->h[5] = 0x9b05688c;
buff->h[6] = 0x1f83d9ab;
buff->h[7] = 0x5be0cd19;
buff->data_size = 0;
buff->chunk_size = 0;
}

static const unsigned int k[64] = {
0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5,
0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174,
0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da,
0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967,
0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85,
0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070,
0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3,
0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2};

#define rotate_r(val, bits) (val >> bits | val << (32 - bits))

static void sha256_calc_chunk(struct sha256_buff* buff, const unsigned char* chunk)
{
unsigned int w[64];
unsigned int tv[8];
unsigned int i;

for (i = 0; i < 16; ++i) {
w[i] = (unsigned int) chunk[0] << 24 | (unsigned int) chunk[1] << 16
| (unsigned int) chunk[2] << 8 | (unsigned int) chunk[3];
chunk += 4;
}

for (i = 16; i < 64; ++i) {
unsigned int s0 = rotate_r(w[i - 15], 7) ^ rotate_r(w[i - 15], 18) ^ (w[i - 15] >> 3);
unsigned int s1 = rotate_r(w[i - 2], 17) ^ rotate_r(w[i - 2], 19) ^ (w[i - 2] >> 10);
w[i] = w[i - 16] + s0 + w[i - 7] + s1;
}

for (i = 0; i < 8; ++i)
tv[i] = buff->h[i];

for (i = 0; i < 64; ++i) {
unsigned int S1 = rotate_r(tv[4], 6) ^ rotate_r(tv[4], 11) ^ rotate_r(tv[4], 25);
unsigned int ch = (tv[4] & tv[5]) ^ (~tv[4] & tv[6]);
unsigned int temp1 = tv[7] + S1 + ch + k[i] + w[i];
unsigned int S0 = rotate_r(tv[0], 2) ^ rotate_r(tv[0], 13) ^ rotate_r(tv[0], 22);
unsigned int maj = (tv[0] & tv[1]) ^ (tv[0] & tv[2]) ^ (tv[1] & tv[2]);
unsigned int temp2 = S0 + maj;

tv[7] = tv[6];
tv[6] = tv[5];
tv[5] = tv[4];
tv[4] = tv[3] + temp1;
tv[3] = tv[2];
tv[2] = tv[1];
tv[1] = tv[0];
tv[0] = temp1 + temp2;
}

for (i = 0; i < 8; ++i)
buff->h[i] += tv[i];
}

void sha256_update(struct sha256_buff* buff, const void* data, unsigned long size)
{
const unsigned char* ptr = (const unsigned char*) data;
buff->data_size += size;
/* If there is data left in buff, concatenate it to process as new chunk */
if (size + buff->chunk_size >= 64) {
unsigned char tmp_chunk[64];
memcpy(tmp_chunk, buff->last_chunk, buff->chunk_size);
memcpy(tmp_chunk + buff->chunk_size, ptr, 64 - buff->chunk_size);
ptr += (64 - buff->chunk_size);
size -= (64 - buff->chunk_size);
buff->chunk_size = 0;
sha256_calc_chunk(buff, tmp_chunk);
}
/* Run over data chunks */
while (size >= 64) {
sha256_calc_chunk(buff, ptr);
ptr += 64;
size -= 64;
}

/* Save remaining data in buff, will be reused on next call or finalize */
memcpy(buff->last_chunk + buff->chunk_size, ptr, size);
buff->chunk_size += size;
}

void sha256_finalize(struct sha256_buff* buff)
{
buff->last_chunk[buff->chunk_size] = 0x80;
buff->chunk_size++;
memset(buff->last_chunk + buff->chunk_size, 0, 64 - buff->chunk_size);

/* If there isn't enough space to fit int64, pad chunk with zeroes and prepare next chunk */
if (buff->chunk_size > 56) {
sha256_calc_chunk(buff, buff->last_chunk);
memset(buff->last_chunk, 0, 64);
}

/* Add total size as big-endian int64 x8 */
unsigned long size = buff->data_size * 8;
int i;
for (i = 8; i > 0; --i) {
buff->last_chunk[55 + i] = size & 255;
size >>= 8;
}

sha256_calc_chunk(buff, buff->last_chunk);
}

void hash_it(const unsigned char* data, unsigned long data_size, unsigned char* hash)
{
struct sha256_buff buff;
sha256_init(&buff);
sha256_update(&buff, data, data_size);
sha256_finalize(&buff);
for (int i = 0; i < 8; i++) {
hash[i * 4] = (buff.h[i] >> 24) & 255;
hash[i * 4 + 1] = (buff.h[i] >> 16) & 255;
hash[i * 4 + 2] = (buff.h[i] >> 8) & 255;
hash[i * 4 + 3] = buff.h[i] & 255;
}
}

}
Loading
Loading