yafcore.h
Go to the documentation of this file.
1/*
2 * Copyright 2006-2023 Carnegie Mellon University
3 * See license information in LICENSE.txt.
4 */
5/*
6 *
7 * yafcore.h
8 * YAF core I/O routines
9 *
10 * ------------------------------------------------------------------------
11 * Authors: Brian Trammell
12 * ------------------------------------------------------------------------
13 * @DISTRIBUTION_STATEMENT_BEGIN@
14 * YAF 2.14.0
15 *
16 * Copyright 2023 Carnegie Mellon University.
17 *
18 * NO WARRANTY. THIS CARNEGIE MELLON UNIVERSITY AND SOFTWARE ENGINEERING
19 * INSTITUTE MATERIAL IS FURNISHED ON AN "AS-IS" BASIS. CARNEGIE MELLON
20 * UNIVERSITY MAKES NO WARRANTIES OF ANY KIND, EITHER EXPRESSED OR IMPLIED,
21 * AS TO ANY MATTER INCLUDING, BUT NOT LIMITED TO, WARRANTY OF FITNESS FOR
22 * PURPOSE OR MERCHANTABILITY, EXCLUSIVITY, OR RESULTS OBTAINED FROM USE OF
23 * THE MATERIAL. CARNEGIE MELLON UNIVERSITY DOES NOT MAKE ANY WARRANTY OF
24 * ANY KIND WITH RESPECT TO FREEDOM FROM PATENT, TRADEMARK, OR COPYRIGHT
25 * INFRINGEMENT.
26 *
27 * Released under a GNU GPL 2.0-style license, please see LICENSE.txt or
28 * contact permission@sei.cmu.edu for full terms.
29 *
30 * [DISTRIBUTION STATEMENT A] This material has been approved for public
31 * release and unlimited distribution. Please see Copyright notice for
32 * non-US Government use and distribution.
33 *
34 * GOVERNMENT PURPOSE RIGHTS - Software and Software Documentation
35 *
36 * Contract No.: FA8702-15-D-0002
37 *
38 * Contractor Name: Carnegie Mellon University
39 *
40 * Contractor Address: 4500 Fifth Avenue, Pittsburgh, PA 15213
41 *
42 * The Government's rights to use, modify, reproduce, release, perform,
43 * display, or disclose this software are restricted by paragraph (b)(2) of
44 * the Rights in Noncommercial Computer Software and Noncommercial Computer
45 * Software Documentation clause contained in the above identified
46 * contract. No restrictions apply after the expiration date shown
47 * above. Any reproduction of the software or portions thereof marked with
48 * this legend must also reproduce the markings.
49 *
50 * Carnegie Mellon(R) and CERT(R) are registered in the U.S. Patent and
51 * Trademark Office by Carnegie Mellon University.
52 *
53 * This Software includes and/or makes use of Third-Party Software each
54 * subject to its own license.
55 *
56 * DM23-0544
57 * @DISTRIBUTION_STATEMENT_END@
58 * ------------------------------------------------------------------------
59 */
60
101#ifndef _YAF_CORE_H_
102#define _YAF_CORE_H_
103
104#include <yaf/autoinc.h>
105#include <fixbuf/public.h>
106#include <stdlib.h>
107#include <math.h>
113#define CERT_PEN 6871
114
120#define YAF_ERROR_DOMAIN (g_quark_from_string("certYAFError"))
122#define YAF_ERROR_HEADER 1
124#define YAF_ERROR_ARGUMENT 2
126#define YAF_ERROR_IO 3
128#define YAF_ERROR_IPFIX 4
130#define YAF_ERROR_IMPL 5
132#define YAF_ERROR_INTERNAL 6
134#define YAF_ERROR_LIMIT 7
136#define YAF_ERROR_EOF 8
138#define YAF_ERROR_ALIGNMENT 9
140#define YAF_ERROR_PACKET_PAYLOAD 10
141
142
143
148#define YAF_FLOW_ACTIVE 0
150#define YAF_END_IDLE 1
152#define YAF_END_ACTIVE 2
154#define YAF_END_CLOSED 3
156#define YAF_END_FORCED 4
158#define YAF_END_RESOURCE 5
160#define YAF_END_UDPFORCE 0x1F
162#define YAF_SAME_SIZE 0x01
164#define YAF_OUT_OF_SEQUENCE 0x02
166#define YAF_MP_CAPABLE 0x04
168#define YAF_FRAGMENTS 0x08
170#define YAF_PARTIAL_FRAGS 0x10
172#define YAF_FRAG_ACTIVE 0x03
174#define YAF_FRAG_PASSIVE 0x04
176#define YAF_END_MASK 0x7F
177
179#define YAF_ENDF_ISCONT 0x80
180
182#define YAF_IP_ICMP 1
184#define YAF_IP_TCP 6
186#define YAF_IP_UDP 17
187
191#define YFP_IPTCPHEADER_SIZE 128
193#define ETHERNET_MAC_ADDR_LENGTH 6
195#define YAF_MAX_HOOKS 4
196
199#define YAF_HOOKS_MAX_EXPORT 1500
201#define YAF_MAX_PKT_BOUNDARY 25
203#define YAF_PCAP_MAX 5000000
205#define YAF_MAX_MPLS_LABELS 3
206
211typedef struct yfFlowKey_st {
213 uint16_t sp;
215 uint16_t dp;
217 uint8_t proto;
219 uint8_t version;
221 uint16_t vlanId;
223 uint8_t tos;
227#if YAF_ENABLE_DAG_SEPARATE_INTERFACES || YAF_ENABLE_SEPARATE_INTERFACES
228 uint8_t netIf;
229#endif
231 union {
232 struct {
234 uint32_t sip;
236 uint32_t dip;
237 } v4;
238 struct {
240 uint8_t sip[16];
242 uint8_t dip[16];
243 } v6;
246
250typedef struct yfFlowStats_st {
252 uint64_t iaarray[10];
254 uint32_t pktsize[10];
256 uint64_t payoct;
258 uint64_t ltime;
260 uint32_t tcpurgct;
262 uint32_t smallpktct;
266 uint32_t largepktct;
268 uint32_t aitime;
270 uint32_t firstpktsize;
272 uint32_t maxpktsize;
274
279typedef struct yfFlowVal_st {
281 uint64_t oct;
283 uint64_t pkt;
284# if YAF_ENABLE_PAYLOAD
286 uint32_t paylen;
288 uint8_t *payload;
290 size_t *paybounds;
291# endif /* if YAF_ENABLE_PAYLOAD */
293 uint32_t isn;
295 uint32_t lsn;
299 uint16_t attributes;
301 uint8_t iflags;
303 uint8_t uflags;
305 uint8_t appkt;
307 uint16_t vlan;
308# if YAF_ENABLE_SEPARATE_INTERFACES
309 uint8_t netIf;
310# endif
311# if YAF_ENABLE_ENTROPY
313 uint8_t entropy;
315 uint8_t entpad[7];
316# endif /* if YAF_ENABLE_ENTROPY */
317# if YAF_ENABLE_P0F
319 const char *osname;
321 const char *osver;
323 uint8_t fuzzyMatch;
325 uint8_t fuzzyPad[7];
327 char *osFingerPrint;
328# endif /* if YAF_ENABLE_P0F */
329# if YAF_ENABLE_FPEXPORT
331 uint32_t firstPacketLen;
333 uint32_t secondPacketLen;
335 uint8_t *firstPacket;
337 uint8_t *secondPacket;
338# endif /* if YAF_ENABLE_FPEXPORT */
342
343#if YAF_MPLS
344typedef struct yfMPLSNode_st {
346 GHashTable *tab;
348 uint32_t mpls_label[YAF_MAX_MPLS_LABELS];
350 int tab_count;
351} yfMPLSNode_t;
352#endif /* if YAF_MPLS */
353
354typedef struct yfMPTCPFlow_st {
356 uint64_t idsn;
358 uint32_t token;
360 uint16_t mss;
362 uint8_t addrid;
364 uint8_t flags;
366
367
374typedef struct yfFlow_st {
376 uint64_t stime;
378 uint64_t etime;
379#ifdef YAF_ENABLE_HOOKS
384 void *hfctx[YAF_MAX_HOOKS];
385#endif
386 /*
387 * Reverse flow delta start time in milliseconds. Equivalent to initial
388 * packet round-trip time; useful for decomposing biflows into uniflows.
389 */
390 int32_t rdtime;
391#if YAF_ENABLE_APPLABEL
393 uint16_t appLabel;
394#endif
395#if YAF_ENABLE_NDPI
396 uint16_t ndpi_master;
397 uint16_t ndpi_sub;
398#endif
400 uint8_t reason;
402 uint8_t pcap_serial;
410 uint8_t pktdir;
412 uint8_t rtos;
414 pcap_dumper_t *pcap;
415#if YAF_MPLS
417 yfMPLSNode_t *mpls;
418#endif
428
437void
439 void);
440
441
449void
451 yfFlow_t *flow);
452
459void
461 yfFlow_t *flow);
462
474fBuf_t *
476 const char *path,
477 uint32_t domain,
478 gboolean export_meta,
479 GError **err);
480
494fBuf_t *
496 FILE *fp,
497 uint32_t domain,
498 gboolean export_meta,
499 GError **err);
500
510fBuf_t *
512 fbConnSpec_t *spec,
513 uint32_t domain,
514 gboolean export_meta,
515 GError **err);
516
517
518#ifdef HAVE_SPREAD
531fBuf_t *
532yfWriterForSpread(
533 fbSpreadParams_t *params,
534 uint32_t domain,
535 uint16_t *spreadGroupIndex,
536 gboolean export_meta,
537 GError **err);
538
539#endif /* HAVE_SPREAD */
540
553gboolean
555 void *yfContext,
556 uint32_t pcap_drop,
557 GTimer *timer,
558 GError **err);
559
574gboolean
576 void *yfContext,
577 uint32_t pcap_drop,
578 GTimer *timer,
579 GError **err);
580
592gboolean
594 void *yfContext,
595 GError **err);
596
608gboolean
610 void *yfContext,
611 yfFlow_t *flow,
612 GError **err);
613
625gboolean
627 fBuf_t *fbuf,
628 gboolean flush,
629 GError **err);
630
634void
636 int max_payload);
637
638#if YAF_ENABLE_APPLABEL
644void
645yfWriterExportPayloadApplabels(
646 const GArray *applabels);
647#endif /* YAF_ENABLE_APPLABEL */
648
652void
654 gboolean map_mode);
655
668fBuf_t *
670 fBuf_t *fbuf,
671 FILE *fp,
672 GError **err);
673
689fbListener_t *
691 fbConnSpec_t *spec,
692 fbListenerAppInit_fn appinit,
693 fbListenerAppFree_fn appfree,
694 GError **err);
695
710gboolean
712 fBuf_t *fbuf,
713 yfFlow_t *flow,
714 GError **err);
715
733gboolean
735 fBuf_t *fbuf,
736 yfFlow_t *flow,
737 GError **err);
738
745void
747 GString *rstr,
748 yfFlow_t *flow);
749
757void
759 GString *rstr,
760 yfFlow_t *flow,
761 gboolean yaft_mac);
762
771gboolean
773 FILE *out,
774 yfFlow_t *flow,
775 GError **err);
776
786gboolean
788 FILE *out,
789 yfFlow_t *flow,
790 gboolean yaft_mac,
791 GError **err);
792
801void
803 FILE *out,
804 gboolean yaft_mac,
805 GError **err);
806
807#if YAF_ENABLE_HOOKS
813fbInfoModel_t *
814yfDPIInfoModel(
815 void);
816
817#endif /* if YAF_ENABLE_HOOKS */
818
819
820
821#endif /* ifndef _YAF_CORE_H_ */
A YAF flow.
Definition: yafcore.h:374
yfFlowVal_t rval
Reverse value.
Definition: yafcore.h:424
uint8_t pcap_serial
Keep track of number of pcap files for this flow.
Definition: yafcore.h:402
uint8_t rtos
reverse ToS (fwd in flowKey)
Definition: yafcore.h:412
uint8_t pktdir
non empty packet directions, 1, or 0
Definition: yafcore.h:410
yfFlowKey_t key
Flow key.
Definition: yafcore.h:426
yfMPTCPFlow_t mptcp
MPTCP Flow.
Definition: yafcore.h:420
uint8_t destinationMacAddr[ETHERNET_MAC_ADDR_LENGTH]
destination Mac Address
Definition: yafcore.h:406
uint8_t sourceMacAddr[ETHERNET_MAC_ADDR_LENGTH]
src Mac Address
Definition: yafcore.h:404
uint64_t stime
Flow start time in epoch milliseconds.
Definition: yafcore.h:376
pcap_dumper_t * pcap
Pcap File Ptr.
Definition: yafcore.h:414
yfFlowVal_t val
Forward value.
Definition: yafcore.h:422
uint8_t pcap_file_no
Pcap File "ID" so we know when to make entries in metadata file.
Definition: yafcore.h:408
uint64_t etime
Flow end time in epoch milliseconds.
Definition: yafcore.h:378
uint8_t reason
Flow termination reason (YAF_END_ macros, per IPFIX standard)
Definition: yafcore.h:400
A YAF flow key.
Definition: yafcore.h:211
uint8_t tos
Type of Service/Traffic Class.
Definition: yafcore.h:223
uint8_t version
IP Version.
Definition: yafcore.h:219
uint32_t sip
Source IPv4 address.
Definition: yafcore.h:234
uint32_t dip
Destination IPv4 address.
Definition: yafcore.h:236
uint16_t sp
Source transport port.
Definition: yafcore.h:213
uint16_t dp
Destination transport port.
Definition: yafcore.h:215
uint16_t vlanId
VLAN Tag - only fwd.
Definition: yafcore.h:221
union yfFlowKey_st::@0 addr
for DAG cards need to record the interface, may only be seeing unidirectional flows on each interface...
uint8_t proto
IP protocol.
Definition: yafcore.h:217
yaf flow statistics
Definition: yafcore.h:250
uint32_t smallpktct
Number of packets with 60 bytes or less of data.
Definition: yafcore.h:262
uint32_t nonemptypktct
total number of non empty pkts
Definition: yafcore.h:264
uint64_t ltime
used to calculate interarrival time
Definition: yafcore.h:258
uint64_t iaarray[10]
to calculate inter-packet delay
Definition: yafcore.h:252
uint32_t maxpktsize
largest pkt size
Definition: yafcore.h:272
uint32_t largepktct
total number of packets with 225 bytes or more
Definition: yafcore.h:266
uint32_t aitime
average interarrival time in milliseconds
Definition: yafcore.h:268
uint32_t firstpktsize
payload length of first non-empty pkt
Definition: yafcore.h:270
uint32_t tcpurgct
Number of urgent packets.
Definition: yafcore.h:260
uint32_t pktsize[10]
to calculate distribution of packet payload size
Definition: yafcore.h:254
uint64_t payoct
total amount of payload data
Definition: yafcore.h:256
A YAF uniflow value.
Definition: yafcore.h:279
uint64_t pkt
Packet count.
Definition: yafcore.h:283
uint32_t lsn
Last TCP sequence number.
Definition: yafcore.h:295
uint8_t appkt
packets with payload - don't care if this wraps.
Definition: yafcore.h:305
uint8_t iflags
Initial TCP flags.
Definition: yafcore.h:301
yfFlowStats_t * stats
yaf flow statistics
Definition: yafcore.h:340
uint64_t oct
Octet count.
Definition: yafcore.h:281
uint16_t first_pkt_size
First Packet Size - to determine whether to turn on fixed size flag.
Definition: yafcore.h:297
uint16_t attributes
flowAttributes
Definition: yafcore.h:299
uint8_t uflags
Union of remaining TCP flags.
Definition: yafcore.h:303
uint16_t vlan
VLAN TAG (also in key, but want to record both sides)
Definition: yafcore.h:307
uint32_t isn
Initial TCP sequence number.
Definition: yafcore.h:293
Definition: yafcore.h:354
uint64_t idsn
initial data seq no.
Definition: yafcore.h:356
uint16_t mss
max segment size
Definition: yafcore.h:360
uint8_t addrid
addr id
Definition: yafcore.h:362
uint32_t token
receiver token
Definition: yafcore.h:358
uint8_t flags
hash_flags
Definition: yafcore.h:364
#define YAF_MAX_MPLS_LABELS
Maximum number of labels we're going to keep around.
Definition: yafcore.h:205
gboolean yfWriteOptionsDataFlows(void *yfContext, uint32_t pcap_drop, GTimer *timer, GError **err)
A wrapper that calls both yfWriteStatsFlow and yfWriteDataFlow with appropriate condition checking.
void yfWriterExportPayload(int max_payload)
FIXME doc.
struct yfFlowKey_st yfFlowKey_t
A YAF flow key.
#define YAF_MAX_HOOKS
maximum number of hooks (plugins) allowed at one time
Definition: yafcore.h:195
gboolean yfReadFlowExtended(fBuf_t *fbuf, yfFlow_t *flow, GError **err)
Read a single flow from an IPFIX message buffer.
fBuf_t * yfWriterForSpec(fbConnSpec_t *spec, uint32_t domain, gboolean export_meta, GError **err)
Get an IPFIX message buffer for writing YAF flows to a socket.
fBuf_t * yfWriterForFP(FILE *fp, uint32_t domain, gboolean export_meta, GError **err)
Get an IPFIX message buffer for writing YAF flows to an open file pointer.
gboolean yfReadFlow(fBuf_t *fbuf, yfFlow_t *flow, GError **err)
Read a single flow from an IPFIX message buffer.
gboolean yfWriteFlow(void *yfContext, yfFlow_t *flow, GError **err)
Write a single flow to an IPFIX message buffer.
void yfPrintString(GString *rstr, yfFlow_t *flow)
Print a YAF flow to a GString.
void yfAlignmentCheck(void)
yfAlignmentCheck
void yfPrintDelimitedString(GString *rstr, yfFlow_t *flow, gboolean yaft_mac)
Print a YAF flow to a GString in pipe-delimited (tabular) format.
void yfFlowCleanup(yfFlow_t *flow)
Clean up after a static flow buffer prepared by yfFlowPrepare.
gboolean yfWriteTombstoneFlow(void *yfContext, GError **err)
Write a tombstone options data record to an IPFIX Message buffer.
fBuf_t * yfWriterForFile(const char *path, uint32_t domain, gboolean export_meta, GError **err)
Get an IPFIX message buffer for writing YAF flows to a named file.
gboolean yfPrint(FILE *out, yfFlow_t *flow, GError **err)
Print a YAF flow to a file.
void yfFlowPrepare(yfFlow_t *flow)
Prepare a static flow buffer for use with yaf_flow_read().
struct yfFlowVal_st yfFlowVal_t
A YAF uniflow value.
#define ETHERNET_MAC_ADDR_LENGTH
length of Ethernet MAC Address
Definition: yafcore.h:193
gboolean yfWriteStatsFlow(void *yfContext, uint32_t pcap_drop, GTimer *timer, GError **err)
Write a statistics options data record to an IPFIX Message buffer.
fBuf_t * yfReaderForFP(fBuf_t *fbuf, FILE *fp, GError **err)
Get an IPFIX message buffer for reading YAF flows from an open file pointer.
gboolean yfPrintDelimited(FILE *out, yfFlow_t *flow, gboolean yaft_mac, GError **err)
Print a YAF flow to a file in pipe-delimited (tabular) format.
struct yfFlow_st yfFlow_t
A YAF flow.
gboolean yfWriterClose(fBuf_t *fbuf, gboolean flush, GError **err)
Close the connection underlying an IPFIX message buffer created by yfWriterForFP() or yfWriterForSpec...
void yfWriterExportMappedV6(gboolean map_mode)
FIXME doc.
void yfPrintColumnHeaders(FILE *out, gboolean yaft_mac, GError **err)
Print column headers for the pipe-delimited (tabular) format.
fbListener_t * yfListenerForSpec(fbConnSpec_t *spec, fbListenerAppInit_fn appinit, fbListenerAppFree_fn appfree, GError **err)
Get an IPFIX connection listener for collecting YAF flows via IPFIX from the network.
struct yfFlowStats_st yfFlowStats_t
yaf flow statistics