yafcore.h
Go to the documentation of this file.
1/*
2 *
3 ** @file yafcore.h
4 ** YAF core I/O routines
5 **
6 ** ------------------------------------------------------------------------
7 ** Copyright (C) 2006-2023 Carnegie Mellon University. All Rights Reserved.
8 ** ------------------------------------------------------------------------
9 ** Authors: Brian Trammell
10 ** ------------------------------------------------------------------------
11 ** Use of the YAF system and related source code is subject to the terms
12 ** of the following licenses:
13 **
14 ** GNU General Public License (GPL) Rights pursuant to Version 2, June 1991
15 ** Government Purpose License Rights (GPLR) pursuant to DFARS 252.227.7013
16 **
17 ** NO WARRANTY
18 **
19 ** ANY INFORMATION, MATERIALS, SERVICES, INTELLECTUAL PROPERTY OR OTHER
20 ** PROPERTY OR RIGHTS GRANTED OR PROVIDED BY CARNEGIE MELLON UNIVERSITY
21 ** PURSUANT TO THIS LICENSE (HEREINAFTER THE "DELIVERABLES") ARE ON AN
22 ** "AS-IS" BASIS. CARNEGIE MELLON UNIVERSITY MAKES NO WARRANTIES OF ANY
23 ** KIND, EITHER EXPRESS OR IMPLIED AS TO ANY MATTER INCLUDING, BUT NOT
24 ** LIMITED TO, WARRANTY OF FITNESS FOR A PARTICULAR PURPOSE,
25 ** MERCHANTABILITY, INFORMATIONAL CONTENT, NONINFRINGEMENT, OR ERROR-FREE
26 ** OPERATION. CARNEGIE MELLON UNIVERSITY SHALL NOT BE LIABLE FOR INDIRECT,
27 ** SPECIAL OR CONSEQUENTIAL DAMAGES, SUCH AS LOSS OF PROFITS OR INABILITY
28 ** TO USE SAID INTELLECTUAL PROPERTY, UNDER THIS LICENSE, REGARDLESS OF
29 ** WHETHER SUCH PARTY WAS AWARE OF THE POSSIBILITY OF SUCH DAMAGES.
30 ** LICENSEE AGREES THAT IT WILL NOT MAKE ANY WARRANTY ON BEHALF OF
31 ** CARNEGIE MELLON UNIVERSITY, EXPRESS OR IMPLIED, TO ANY PERSON
32 ** CONCERNING THE APPLICATION OF OR THE RESULTS TO BE OBTAINED WITH THE
33 ** DELIVERABLES UNDER THIS LICENSE.
34 **
35 ** Licensee hereby agrees to defend, indemnify, and hold harmless Carnegie
36 ** Mellon University, its trustees, officers, employees, and agents from
37 ** all claims or demands made against them (and any related losses,
38 ** expenses, or attorney's fees) arising out of, or relating to Licensee's
39 ** and/or its sub licensees' negligent use or willful misuse of or
40 ** negligent conduct or willful misconduct regarding the Software,
41 ** facilities, or other rights or assistance granted by Carnegie Mellon
42 ** University under this License, including, but not limited to, any
43 ** claims of product liability, personal injury, death, damage to
44 ** property, or violation of any laws or regulations.
45 **
46 ** Carnegie Mellon University Software Engineering Institute authored
47 ** documents are sponsored by the U.S. Department of Defense under
48 ** Contract FA8721-05-C-0003. Carnegie Mellon University retains
49 ** copyrights in all material produced under this contract. The U.S.
50 ** Government retains a non-exclusive, royalty-free license to publish or
51 ** reproduce these documents, or allow others to do so, for U.S.
52 ** Government purposes only pursuant to the copyright license under the
53 ** contract clause at 252.227.7013.
54 **
55 ** ------------------------------------------------------------------------
56 */
57
98#ifndef _YAF_CORE_H_
99#define _YAF_CORE_H_
100
101#include <yaf/autoinc.h>
102#include <fixbuf/public.h>
103#include <stdlib.h>
104#include <math.h>
110#define CERT_PEN 6871
111
117#define YAF_ERROR_DOMAIN (g_quark_from_string("certYAFError"))
119#define YAF_ERROR_HEADER 1
121#define YAF_ERROR_ARGUMENT 2
123#define YAF_ERROR_IO 3
125#define YAF_ERROR_IPFIX 4
127#define YAF_ERROR_IMPL 5
129#define YAF_ERROR_INTERNAL 6
131#define YAF_ERROR_LIMIT 7
133#define YAF_ERROR_EOF 8
135#define YAF_ERROR_ALIGNMENT 9
137#define YAF_ERROR_PACKET_PAYLOAD 10
138
139
140
145#define YAF_FLOW_ACTIVE 0
147#define YAF_END_IDLE 1
149#define YAF_END_ACTIVE 2
151#define YAF_END_CLOSED 3
153#define YAF_END_FORCED 4
155#define YAF_END_RESOURCE 5
157#define YAF_END_UDPFORCE 0x1F
159#define YAF_SAME_SIZE 0x01
161#define YAF_OUT_OF_SEQUENCE 0x02
163#define YAF_MP_CAPABLE 0x04
165#define YAF_FRAGMENTS 0x08
167#define YAF_PARTIAL_FRAGS 0x10
169#define YAF_FRAG_ACTIVE 0x03
171#define YAF_FRAG_PASSIVE 0x04
173#define YAF_END_MASK 0x7F
174
176#define YAF_ENDF_ISCONT 0x80
177
179#define YAF_IP_ICMP 1
181#define YAF_IP_TCP 6
183#define YAF_IP_UDP 17
184
188#define YFP_IPTCPHEADER_SIZE 128
190#define ETHERNET_MAC_ADDR_LENGTH 6
192#define YAF_MAX_HOOKS 4
193
196#define YAF_HOOKS_MAX_EXPORT 1500
198#define YAF_MAX_PKT_BOUNDARY 25
200#define YAF_PCAP_MAX 5000000
202#define YAF_MAX_MPLS_LABELS 3
203
208typedef struct yfFlowKey_st {
210 uint16_t sp;
212 uint16_t dp;
214 uint8_t proto;
216 uint8_t version;
218 uint16_t vlanId;
220 uint8_t tos;
224#if YAF_ENABLE_DAG_SEPARATE_INTERFACES || YAF_ENABLE_SEPARATE_INTERFACES
225 uint8_t netIf;
226#endif
228 union {
229 struct {
231 uint32_t sip;
233 uint32_t dip;
234 } v4;
235 struct {
237 uint8_t sip[16];
239 uint8_t dip[16];
240 } v6;
243
247typedef struct yfFlowStats_st {
249 uint64_t iaarray[10];
251 uint32_t pktsize[10];
253 uint64_t payoct;
255 uint64_t ltime;
257 uint32_t tcpurgct;
259 uint32_t smallpktct;
263 uint32_t largepktct;
265 uint32_t aitime;
267 uint32_t firstpktsize;
269 uint32_t maxpktsize;
271
276typedef struct yfFlowVal_st {
278 uint64_t oct;
280 uint64_t pkt;
281# if YAF_ENABLE_PAYLOAD
283 uint32_t paylen;
285 uint8_t *payload;
287 size_t *paybounds;
288# endif /* if YAF_ENABLE_PAYLOAD */
290 uint32_t isn;
292 uint32_t lsn;
296 uint16_t attributes;
298 uint8_t iflags;
300 uint8_t uflags;
302 uint8_t appkt;
304 uint16_t vlan;
305# if YAF_ENABLE_SEPARATE_INTERFACES
306 uint8_t netIf;
307# endif
308# if YAF_ENABLE_ENTROPY
310 uint8_t entropy;
312 uint8_t entpad[7];
313# endif /* if YAF_ENABLE_ENTROPY */
314# if YAF_ENABLE_P0F
316 const char *osname;
318 const char *osver;
320 uint8_t fuzzyMatch;
322 uint8_t fuzzyPad[7];
324 char *osFingerPrint;
325# endif /* if YAF_ENABLE_P0F */
326# if YAF_ENABLE_FPEXPORT
328 uint32_t firstPacketLen;
330 uint32_t secondPacketLen;
332 uint8_t *firstPacket;
334 uint8_t *secondPacket;
335# endif /* if YAF_ENABLE_FPEXPORT */
339
340#if YAF_MPLS
341typedef struct yfMPLSNode_st {
343 GHashTable *tab;
345 uint32_t mpls_label[YAF_MAX_MPLS_LABELS];
347 int tab_count;
348} yfMPLSNode_t;
349#endif /* if YAF_MPLS */
350
351typedef struct yfMPTCPFlow_st {
353 uint64_t idsn;
355 uint32_t token;
357 uint16_t mss;
359 uint8_t addrid;
361 uint8_t flags;
363
364
371typedef struct yfFlow_st {
373 uint64_t stime;
375 uint64_t etime;
376#ifdef YAF_ENABLE_HOOKS
381 void *hfctx[YAF_MAX_HOOKS];
382#endif
383 /*
384 * Reverse flow delta start time in milliseconds. Equivalent to initial
385 * packet round-trip time; useful for decomposing biflows into uniflows.
386 */
387 int32_t rdtime;
388#if YAF_ENABLE_APPLABEL
390 uint16_t appLabel;
391#endif
392#if YAF_ENABLE_NDPI
393 uint16_t ndpi_master;
394 uint16_t ndpi_sub;
395#endif
397 uint8_t reason;
399 uint8_t pcap_serial;
407 uint8_t pktdir;
409 uint8_t rtos;
411 pcap_dumper_t *pcap;
412#if YAF_MPLS
414 yfMPLSNode_t *mpls;
415#endif
425
434void
436 void);
437
438
446void
448 yfFlow_t *flow);
449
456void
458 yfFlow_t *flow);
459
471fBuf_t *
473 const char *path,
474 uint32_t domain,
475 gboolean export_meta,
476 GError **err);
477
491fBuf_t *
493 FILE *fp,
494 uint32_t domain,
495 gboolean export_meta,
496 GError **err);
497
507fBuf_t *
509 fbConnSpec_t *spec,
510 uint32_t domain,
511 gboolean export_meta,
512 GError **err);
513
514
515#ifdef HAVE_SPREAD
528fBuf_t *
529yfWriterForSpread(
530 fbSpreadParams_t *params,
531 uint32_t domain,
532 uint16_t *spreadGroupIndex,
533 gboolean export_meta,
534 GError **err);
535
536#endif /* HAVE_SPREAD */
537
550gboolean
552 void *yfContext,
553 uint32_t pcap_drop,
554 GTimer *timer,
555 GError **err);
556
571gboolean
573 void *yfContext,
574 uint32_t pcap_drop,
575 GTimer *timer,
576 GError **err);
577
589gboolean
591 void *yfContext,
592 GError **err);
593
605gboolean
607 void *yfContext,
608 yfFlow_t *flow,
609 GError **err);
610
622gboolean
624 fBuf_t *fbuf,
625 gboolean flush,
626 GError **err);
627
631void
633 int max_payload);
634
635#if YAF_ENABLE_APPLABEL
641void
642yfWriterExportPayloadApplabels(
643 const GArray *applabels);
644#endif /* YAF_ENABLE_APPLABEL */
645
649void
651 gboolean map_mode);
652
665fBuf_t *
667 fBuf_t *fbuf,
668 FILE *fp,
669 GError **err);
670
686fbListener_t *
688 fbConnSpec_t *spec,
689 fbListenerAppInit_fn appinit,
690 fbListenerAppFree_fn appfree,
691 GError **err);
692
707gboolean
709 fBuf_t *fbuf,
710 yfFlow_t *flow,
711 GError **err);
712
730gboolean
732 fBuf_t *fbuf,
733 yfFlow_t *flow,
734 GError **err);
735
742void
744 GString *rstr,
745 yfFlow_t *flow);
746
754void
756 GString *rstr,
757 yfFlow_t *flow,
758 gboolean yaft_mac);
759
768gboolean
770 FILE *out,
771 yfFlow_t *flow,
772 GError **err);
773
783gboolean
785 FILE *out,
786 yfFlow_t *flow,
787 gboolean yaft_mac,
788 GError **err);
789
798void
800 FILE *out,
801 gboolean yaft_mac,
802 GError **err);
803
804#if YAF_ENABLE_HOOKS
810fbInfoModel_t *
811yfDPIInfoModel(
812 void);
813
814#endif /* if YAF_ENABLE_HOOKS */
815
816
817
818#endif /* ifndef _YAF_CORE_H_ */
A YAF flow.
Definition: yafcore.h:371
yfFlowVal_t rval
Reverse value.
Definition: yafcore.h:421
uint8_t pcap_serial
Keep track of number of pcap files for this flow.
Definition: yafcore.h:399
uint8_t rtos
reverse ToS (fwd in flowKey)
Definition: yafcore.h:409
uint8_t pktdir
non empty packet directions, 1, or 0
Definition: yafcore.h:407
yfFlowKey_t key
Flow key.
Definition: yafcore.h:423
yfMPTCPFlow_t mptcp
MPTCP Flow.
Definition: yafcore.h:417
uint8_t destinationMacAddr[ETHERNET_MAC_ADDR_LENGTH]
destination Mac Address
Definition: yafcore.h:403
uint8_t sourceMacAddr[ETHERNET_MAC_ADDR_LENGTH]
src Mac Address
Definition: yafcore.h:401
uint64_t stime
Flow start time in epoch milliseconds.
Definition: yafcore.h:373
pcap_dumper_t * pcap
Pcap File Ptr.
Definition: yafcore.h:411
yfFlowVal_t val
Forward value.
Definition: yafcore.h:419
uint8_t pcap_file_no
Pcap File "ID" so we know when to make entries in metadata file.
Definition: yafcore.h:405
uint64_t etime
Flow end time in epoch milliseconds.
Definition: yafcore.h:375
uint8_t reason
Flow termination reason (YAF_END_ macros, per IPFIX standard)
Definition: yafcore.h:397
A YAF flow key.
Definition: yafcore.h:208
uint8_t tos
Type of Service/Traffic Class.
Definition: yafcore.h:220
uint8_t version
IP Version.
Definition: yafcore.h:216
uint32_t sip
Source IPv4 address.
Definition: yafcore.h:231
uint32_t dip
Destination IPv4 address.
Definition: yafcore.h:233
uint16_t sp
Source transport port.
Definition: yafcore.h:210
uint16_t dp
Destination transport port.
Definition: yafcore.h:212
uint16_t vlanId
VLAN Tag - only fwd.
Definition: yafcore.h:218
union yfFlowKey_st::@0 addr
for DAG cards need to record the interface, may only be seeing unidirectional flows on each interface...
uint8_t proto
IP protocol.
Definition: yafcore.h:214
yaf flow statistics
Definition: yafcore.h:247
uint32_t smallpktct
Number of packets with 60 bytes or less of data.
Definition: yafcore.h:259
uint32_t nonemptypktct
total number of non empty pkts
Definition: yafcore.h:261
uint64_t ltime
used to calculate interarrival time
Definition: yafcore.h:255
uint64_t iaarray[10]
to calculate inter-packet delay
Definition: yafcore.h:249
uint32_t maxpktsize
largest pkt size
Definition: yafcore.h:269
uint32_t largepktct
total number of packets with 225 bytes or more
Definition: yafcore.h:263
uint32_t aitime
average interarrival time in milliseconds
Definition: yafcore.h:265
uint32_t firstpktsize
payload length of first non-empty pkt
Definition: yafcore.h:267
uint32_t tcpurgct
Number of urgent packets.
Definition: yafcore.h:257
uint32_t pktsize[10]
to calculate distribution of packet payload size
Definition: yafcore.h:251
uint64_t payoct
total amount of payload data
Definition: yafcore.h:253
A YAF uniflow value.
Definition: yafcore.h:276
uint64_t pkt
Packet count.
Definition: yafcore.h:280
uint32_t lsn
Last TCP sequence number.
Definition: yafcore.h:292
uint8_t appkt
packets with payload - don't care if this wraps.
Definition: yafcore.h:302
uint8_t iflags
Initial TCP flags.
Definition: yafcore.h:298
yfFlowStats_t * stats
yaf flow statistics
Definition: yafcore.h:337
uint64_t oct
Octet count.
Definition: yafcore.h:278
uint16_t first_pkt_size
First Packet Size - to determine whether to turn on fixed size flag.
Definition: yafcore.h:294
uint16_t attributes
flowAttributes
Definition: yafcore.h:296
uint8_t uflags
Union of remaining TCP flags.
Definition: yafcore.h:300
uint16_t vlan
VLAN TAG (also in key, but want to record both sides)
Definition: yafcore.h:304
uint32_t isn
Initial TCP sequence number.
Definition: yafcore.h:290
Definition: yafcore.h:351
uint64_t idsn
initial data seq no.
Definition: yafcore.h:353
uint16_t mss
max segment size
Definition: yafcore.h:357
uint8_t addrid
addr id
Definition: yafcore.h:359
uint32_t token
receiver token
Definition: yafcore.h:355
uint8_t flags
hash_flags
Definition: yafcore.h:361
#define YAF_MAX_MPLS_LABELS
Maximum number of labels we're going to keep around.
Definition: yafcore.h:202
gboolean yfWriteOptionsDataFlows(void *yfContext, uint32_t pcap_drop, GTimer *timer, GError **err)
A wrapper that calls both yfWriteStatsFlow and yfWriteDataFlow with appropriate condition checking.
void yfWriterExportPayload(int max_payload)
FIXME doc.
struct yfFlowKey_st yfFlowKey_t
A YAF flow key.
#define YAF_MAX_HOOKS
maximum number of hooks (plugins) allowed at one time
Definition: yafcore.h:192
gboolean yfReadFlowExtended(fBuf_t *fbuf, yfFlow_t *flow, GError **err)
Read a single flow from an IPFIX message buffer.
fBuf_t * yfWriterForSpec(fbConnSpec_t *spec, uint32_t domain, gboolean export_meta, GError **err)
Get an IPFIX message buffer for writing YAF flows to a socket.
fBuf_t * yfWriterForFP(FILE *fp, uint32_t domain, gboolean export_meta, GError **err)
Get an IPFIX message buffer for writing YAF flows to an open file pointer.
gboolean yfReadFlow(fBuf_t *fbuf, yfFlow_t *flow, GError **err)
Read a single flow from an IPFIX message buffer.
gboolean yfWriteFlow(void *yfContext, yfFlow_t *flow, GError **err)
Write a single flow to an IPFIX message buffer.
void yfPrintString(GString *rstr, yfFlow_t *flow)
Print a YAF flow to a GString.
void yfAlignmentCheck(void)
yfAlignmentCheck
void yfPrintDelimitedString(GString *rstr, yfFlow_t *flow, gboolean yaft_mac)
Print a YAF flow to a GString in pipe-delimited (tabular) format.
void yfFlowCleanup(yfFlow_t *flow)
Clean up after a static flow buffer prepared by yfFlowPrepare.
gboolean yfWriteTombstoneFlow(void *yfContext, GError **err)
Write a tombstone options data record to an IPFIX Message buffer.
fBuf_t * yfWriterForFile(const char *path, uint32_t domain, gboolean export_meta, GError **err)
Get an IPFIX message buffer for writing YAF flows to a named file.
gboolean yfPrint(FILE *out, yfFlow_t *flow, GError **err)
Print a YAF flow to a file.
void yfFlowPrepare(yfFlow_t *flow)
Prepare a static flow buffer for use with yaf_flow_read().
struct yfFlowVal_st yfFlowVal_t
A YAF uniflow value.
#define ETHERNET_MAC_ADDR_LENGTH
length of Ethernet MAC Address
Definition: yafcore.h:190
gboolean yfWriteStatsFlow(void *yfContext, uint32_t pcap_drop, GTimer *timer, GError **err)
Write a statistics options data record to an IPFIX Message buffer.
fBuf_t * yfReaderForFP(fBuf_t *fbuf, FILE *fp, GError **err)
Get an IPFIX message buffer for reading YAF flows from an open file pointer.
gboolean yfPrintDelimited(FILE *out, yfFlow_t *flow, gboolean yaft_mac, GError **err)
Print a YAF flow to a file in pipe-delimited (tabular) format.
struct yfFlow_st yfFlow_t
A YAF flow.
gboolean yfWriterClose(fBuf_t *fbuf, gboolean flush, GError **err)
Close the connection underlying an IPFIX message buffer created by yfWriterForFP() or yfWriterForSpec...
void yfWriterExportMappedV6(gboolean map_mode)
FIXME doc.
void yfPrintColumnHeaders(FILE *out, gboolean yaft_mac, GError **err)
Print column headers for the pipe-delimited (tabular) format.
fbListener_t * yfListenerForSpec(fbConnSpec_t *spec, fbListenerAppInit_fn appinit, fbListenerAppFree_fn appfree, GError **err)
Get an IPFIX connection listener for collecting YAF flows via IPFIX from the network.
struct yfFlowStats_st yfFlowStats_t
yaf flow statistics