Changeset b663d33

Timestamp:

05/18/18 11:39:59 (3 years ago)

Author:

Shane Alcock <salcock@…>

Branches:

cachetimestamps, develop, etsilive, master, rc-4.0.4, ringdecrementfix, ringperformance

Children:

00365c6

Parents:

df87f00

git-author:

Shane Alcock <salcock@…> (12/18/17 18:10:37)

git-committer:

Shane Alcock <salcock@…> (05/18/18 11:39:59)

Message:

etsilive format is now functional (for single-threaded only).

Packets are decoded using libwandder. The ETSI headers are treated
as a meta-data layer.

Libpacketdump support has also been added for all fields and
structures that libwandder understands.

Files:

• configure.in (modified) (5 diffs)
• lib/Makefile.am (modified) (2 diffs)
• lib/format_etsilive.c (modified) (15 diffs)
• lib/linktypes.c (modified) (2 diffs)
• lib/protocols_l2.c (modified) (5 diffs)
• lib/protocols_pktmeta.c (modified) (4 diffs)
• lib/trace.c (modified) (1 diff)
• libpacketdump/Makefile.am (modified) (2 diffs)
• libpacketdump/link_21.c (added)

configure.in

@@ -459 +459 @@
 AC_CHECK_LIB(numa, numa_node_to_cpus, have_numa=1, have_numa=0)
 
+# Need libwandder for ETSI live decoding
+AC_CHECK_LIB(wandder, init_wandder_decoder, have_wandder=1, have_wandder=0)
+
 # Checks for various "optional" libraries
 AC_CHECK_LIB(pthread, pthread_create, have_pthread=1, have_pthread=0)
@@ -490 +493 @@
 fi
 
+if test "$have_wandder" = 1; then
+        LIBTRACE_LIBS="$LIBTRACE_LIBS -lwandder"
+        AC_DEFINE(HAVE_WANDDER, 1, [Set to 1 if libwandder is available])
+        wandder_avail=yes
+else
+        AC_DEFINE(HAVE_WANDDER, 0, [Set to 1 if libwandder is available])
+        wandder_avail=no
+fi
+
 if test "$dlfound" = 0; then
         AC_MSG_ERROR("Unable to find dlopen. Please use LDFLAGS to specify the location of libdl and re-run configure")
@@ -515 +527 @@
 else
         have_crypto=no
-fi
-
-if test "$asn1found" = 1; then
-        AC_DEFINE(HAVE_LIBTASN1, 1, [Set to 1 if libtasn1 is available])
-        LIBTRACE_LIBS="$LIBTRACE_LIBS -ltasn1"
-        TOOLS_LIBS="$TOOLS_LIBS -ltasn1"
-        have_asn1=yes
-else
-        have_asn1=no
 fi
 
@@ -624 +627 @@
 AM_CONDITIONAL([HAVE_DAG], [test "$libtrace_dag" = true])
 AM_CONDITIONAL([HAVE_DPDK], [test "$libtrace_dpdk" = true])
+AM_CONDITIONAL([HAVE_WANDDER], [test "x$wandder_avail" = "xyes"])
 AM_CONDITIONAL([DAG2_4], [test "$libtrace_dag_version" = 24])
 AM_CONDITIONAL([DAG2_5], [test "$libtrace_dag_version" = 25])
@@ -705 +709 @@
 fi
 reportopt "Compiled with LLVM BPF JIT support" $JIT
-reportopt "Compiled with live ETSI LI support (requires libtasn1)" $have_asn1
+reportopt "Compiled with live ETSI LI support (requires libwandder)" $wandder_avail
 reportopt "Building man pages/documentation" $libtrace_doxygen
 reportopt "Building tracetop (requires libncurses)" $with_ncurses

lib/Makefile.am

@@ -52 +52 @@
 endif
 
+if HAVE_WANDDER
+ETSISOURCES=format_etsilive.c
+else
+ETSISOURCES=
+endif
+
 libtrace_la_SOURCES = trace.c trace_parallel.c common.h \
                 format_erf.c format_pcap.c format_legacy.c \
                 format_rt.c format_helper.c format_helper.h format_pcapfile.c \
                 format_duck.c format_tsh.c $(NATIVEFORMATS) $(BPFFORMATS) \
-                format_atmhdr.c format_pcapng.c format_etsilive.c \
+                format_atmhdr.c format_pcapng.c \
                 libtrace_int.h lt_inttypes.h lt_bswap.h \
                 linktypes.c link_wireless.c byteswap.c \
@@ -64 +70 @@
                 protocols_application.c \
                 $(DAGSOURCE) format_erf.h format_ndag.c format_ndag.h \
-                $(BPFJITSOURCE) \
+                $(BPFJITSOURCE) $(ETSISOURCES) \
                 libtrace_arphrd.h \
                 data-struct/ring_buffer.c data-struct/vector.c \

lib/format_etsilive.c

@@ -34 +34 @@
 #include "data-struct/simple_circular_buffer.h"
 
-#include "etsiasn1tab.h"
+#include <libwandder.h>
+#include <libwandder_etsili.h>
 
 #include <assert.h>
@@ -43 +44 @@
 #include <stdlib.h>
 #include <unistd.h>
-#include <libtasn1.h>
 #include <sys/socket.h>
 #include <sys/types.h>
@@ -55 +55 @@
 
         uint64_t timestamp;
-        uint8_t *ccptr;
-        uint8_t *iriptr;
         uint16_t length;
 
@@ -84 +82 @@
         etsithread_t *receivers;
         int nextthreadid;
-        ASN1_TYPE etsidef;
 } etsilive_format_data_t;
 
@@ -210 +207 @@
 static int etsilive_init_input(libtrace_t *libtrace) {
         char *scan = NULL;
-        char errordesc[ASN1_MAX_ERROR_DESCRIPTION_SIZE];
-
         libtrace->format_data = (etsilive_format_data_t *)malloc(
                         sizeof(etsilive_format_data_t));
@@ -230 +225 @@
         FORMAT_DATA->listenport = strdup(scan + 1);
 
-        if (asn1_array2tree(etsili_asn1tab, &(FORMAT_DATA->etsidef),
-                                errordesc) != ASN1_SUCCESS) {
-                trace_set_err(libtrace, TRACE_ERR_BAD_FORMAT,
-                        "Failed to parse ETSI ASN.1 array: %s", errordesc);
-                return -1;
-        }
-
         return 0;
 }
@@ -251 +239 @@
                 free(FORMAT_DATA->listenport);
         }
-        asn1_delete_structure(&(FORMAT_DATA->etsidef));
         free(libtrace->format_data);
         return 0;
@@ -339 +326 @@
                 esock->cached.timestamp = 0;
                 esock->cached.length = 0;
-                esock->cached.iriptr = NULL;
-                esock->cached.ccptr = NULL;
 
                 et->sourcecount += 1;
@@ -406 +391 @@
 static etsisocket_t *select_next_packet(etsithread_t *et, libtrace_t *libtrace) {
 
-        int i, asnret;
+        int i;
         etsisocket_t *esock = NULL;
-        char errordesc[ASN1_MAX_ERROR_DESCRIPTION_SIZE];
-        char readstring[100];
         uint64_t earliest = 0;
-        uint64_t currentts = 0;
+        uint64_t current;
+        struct timeval tv;
         uint32_t available;
         uint8_t *ptr = NULL;
-        ASN1_TYPE psheader;
-        int ider_len;
-        int readlen;
+        wandder_decoder_t dec;
+        uint32_t reclen = 0;
 
         for (i = 0; i < et->sourcecount; i++) {
@@ -426 +409 @@
                  */
                 if (et->sources[i].cached.timestamp != 0) {
-                        currentts = et->sources[i].cached.timestamp;
-
-                        if (earliest == 0 || earliest > currentts) {
-                                earliest = currentts;
+                        current = et->sources[i].cached.timestamp;
+
+                        if (earliest == 0 || earliest > current) {
+                                earliest = current;
                                 esock = &(et->sources[i]);
                         }
@@ -438 +421 @@
                                 &available);
 
-                if (available == 0) {
+                if (available == 0 || ptr == NULL) {
                         continue;
                 }
 
-                /* Try to decode whatever is at the front of the buffer. */
-                asnret = asn1_create_element(FORMAT_DATA->etsidef,
-                                "LI-PS-PDU.PSHeader", &psheader);
-                if (asnret != ASN1_SUCCESS) {
-                        fprintf(stderr, "failed to create asn1 element\n");
-                        asn1_delete_structure(&psheader);
-                        continue;
-                }
-
-                ider_len = (int)available;
-                asnret = asn1_der_decoding2(&psheader, ptr, &ider_len, 0, errordesc);
-
-                /* Failed? Must not have the whole packet... */
-                if (asnret != ASN1_SUCCESS) {
-                        int j;
-                        for (j = 0; j < available; j++) {
-                                printf("%02x ", ptr[j]);
-                                if (j % 16 == 15 && j > 0) {
-                                        printf("\n");
-                                }
-                                if (j >= 16 * 16)
-                                        break;
-                        }
-                        fprintf(stderr, "%d failed to decode asn1 content: %s\n",
-                                        available, errordesc);
-                        assert(0);
-                        asn1_delete_structure(&psheader);
-                        continue;
-                }
-
-                readlen = sizeof(readstring);
-                asnret = asn1_read_value(psheader, "timeStamp", readstring, &readlen);
-
-                if (asnret == ASN1_SUCCESS) {
-                        fprintf(stderr, "timeStamp=%s\n", readstring);
-                        /* TODO turn to 64 bit timestamp */
+                init_wandder_decoder(&dec, ptr, available, false);
+                if (et->sources[i].cached.length != 0) {
+                        reclen = et->sources[i].cached.length;
                 } else {
-                        int msts_sec;
-                        int msts_ms;
-
-                        readlen = sizeof(int);
-                        asnret = asn1_read_value(psheader, "microSecondTimeStamp.seconds", &msts_sec, &readlen);
-
-                        if (asnret != ASN1_SUCCESS) {
-                                fprintf(stderr, "no microSecondTimeStamp.seconds\n");
+                        reclen = wandder_etsili_get_pdu_length(&dec);
+
+                        if (reclen == 0) {
+                                free_wandder_decoder(&dec);
                                 continue;
                         }
-
-                        readlen = sizeof(int);
-                        asnret = asn1_read_value(psheader, "microSecondTimeStamp.microSeconds", &msts_ms, &readlen);
-
-                        if (asnret != ASN1_SUCCESS) {
-                                fprintf(stderr, "no microSecondTimeStamp.microSeconds?\n");
-                                continue;
-                        }
-                        fprintf(stderr, "microSecondTimeStamp=%d.%d\n", msts_sec, msts_ms);
-                }
-                assert(0);
+                }
+
+                if (available < reclen) {
+                        /* Don't have the whole PDU yet */
+                        free_wandder_decoder(&dec);
+                        continue;
+                }
+
+                /* Get the timestamp */
+
+                tv = wandder_etsili_get_header_timestamp(&dec);
+                if (tv.tv_sec == 0) {
+                        free_wandder_decoder(&dec);
+                        continue;
+                }
+                current = ((((uint64_t)tv.tv_sec) << 32) +
+                                (((uint64_t)tv.tv_usec << 32)/1000000)); 
+
                 /* Success, cache everything we used so we don't have to
                  * decode this packet again.
                  */
-
-                /* Advance the read pointer for this buffer */
+                et->sources[i].cached.timestamp = current;
+                et->sources[i].cached.length = reclen;
+
 
                 /* Don't forget to update earliest and esock... */
+                if (current < earliest || earliest == 0) {
+                        esock = &(et->sources[i]);
+                        earliest = current;
+                }
 
         }
@@ -514 +472 @@
 static int etsilive_prepare_received(libtrace_t *libtrace, etsithread_t *et,
                 etsisocket_t *esock, libtrace_packet_t *packet) {
+
+        uint32_t available = 0;
+
+        packet->trace = libtrace;
+        packet->buffer = libtrace_scb_get_read(&(esock->recvbuffer),
+                                        &available);
+        packet->buf_control = TRACE_CTRL_EXTERNAL;
+        packet->header = NULL;          // Check this is ok to do
+        packet->payload = packet->buffer;
+        packet->type = TRACE_RT_DATA_ETSILI;
+        packet->order = esock->cached.timestamp;
+        packet->error = esock->cached.length;
+
+        packet->wire_length = esock->cached.length;
+        packet->capture_length = esock->cached.length;
+
+        /* Advance the read pointer for this buffer
+         * TODO should really do this in fin_packet, but will need a ref
+         * to esock to do this properly */
+        libtrace_scb_advance_read(&(esock->recvbuffer), esock->cached.length);
+        esock->cached.length = 0;
+        esock->cached.timestamp = 0;
+
+
         return 1;
 }
@@ -562 +544 @@
 }
 
-
-
+static int etsilive_get_pdu_length(const libtrace_packet_t *packet) {
+
+        /* Should never get here because cache is set when packet is read */
+        wandder_decoder_t dec;
+        size_t reclen;
+
+        /* 0 should be ok here for quickly evaluating the first length
+         * field... */
+        init_wandder_decoder(&dec, packet->buffer, 0, false);
+        reclen = (size_t)wandder_etsili_get_pdu_length(&dec);
+
+        free_wandder_decoder(&dec);
+
+        return reclen;
+}
+
+static int etsilive_get_framing_length(const libtrace_packet_t *packet) {
+
+        return 0;
+}
+
+static struct timeval etsilive_get_timeval(const libtrace_packet_t *packet) {
+        /* TODO add cached timestamps to libtrace so we don't have to look
+         * this up again. */
+        struct timeval tv;
+        wandder_decoder_t dec;
+
+        tv.tv_sec = 0;
+        tv.tv_usec = 0;
+
+        init_wandder_decoder(&dec, packet->buffer, 0, false);
+        tv = wandder_etsili_get_header_timestamp(&dec);
+        free_wandder_decoder(&dec);
+        return tv;
+}
+
+static libtrace_linktype_t etsilive_get_link_type(
+                const libtrace_packet_t *packet) {
+        return TRACE_TYPE_ETSILI;
+}
 
 static struct libtrace_format_t etsilive = {
@@ -584 +604 @@
         NULL,                           /* fin_packet */
         NULL,                           /* write_packet */
-        NULL, //etsilive_get_link_type,         /* get_link_type */
+        etsilive_get_link_type,         /* get_link_type */
         NULL,                           /* get_direction */
         NULL,                           /* set_direction */
         NULL,                           /* get_erf_timestamp */
-        NULL,                           /* get_timeval */
+        etsilive_get_timeval,           /* get_timeval */
         NULL,                           /* get_timespec */
         NULL,                           /* get_seconds */
@@ -594 +614 @@
         NULL,                           /* seek_timeval */
         NULL,                           /* seek_seconds */
-        NULL, //etsilive_get_capture_length,    /* get_capture_length */
-        NULL, //etsilive_get_wire_length,       /* get_wire_length */
-        NULL, //etsilive_get_framing_length,    /* get_framing_length */
+        etsilive_get_pdu_length,       /* get_capture_length */
+        etsilive_get_pdu_length,       /* get_wire_length */
+        etsilive_get_framing_length,    /* get_framing_length */
         NULL,                           /* set_capture_length */
         NULL,                           /* get_received_packets */

lib/linktypes.c

@@ -109 +109 @@
                 case TRACE_TYPE_METADATA:
                 case TRACE_TYPE_NONDATA:
+                case TRACE_TYPE_ETSILI:
                         break;
                 case TRACE_TYPE_UNKNOWN:
@@ -211 +212 @@
                 case TRACE_TYPE_NONDATA:
                 case TRACE_TYPE_OPENBSD_LOOP:
+                case TRACE_TYPE_ETSILI:
                 case TRACE_TYPE_UNKNOWN:
                         break;

lib/protocols_l2.c

@@ -483 +483 @@
                 case TRACE_TYPE_PFLOG:
                 case TRACE_TYPE_ERF_META:
+                case TRACE_TYPE_ETSILI:
                         break;
                 case TRACE_TYPE_UNKNOWN:
@@ -518 +519 @@
                                 case TRACE_TYPE_80211_PRISM:
                                 case TRACE_TYPE_PFLOG:
+<<<<<<< HEAD
                                 case TRACE_TYPE_ERF_META:
+=======
+                                case TRACE_TYPE_ETSILI:
+>>>>>>> etsilive format is now functional (for single-threaded only).
                                         break;
                                 case TRACE_TYPE_UNKNOWN:
@@ -574 +579 @@
                 case TRACE_TYPE_PFLOG:
                 case TRACE_TYPE_LINUX_SLL:
+                case TRACE_TYPE_ETSILI:
                         return NULL;
 
@@ -696 +702 @@
                 case TRACE_TYPE_80211_PRISM:
                 case TRACE_TYPE_80211_RADIO:
+                case TRACE_TYPE_ETSILI:
                         assert(!"Metadata headers should already be skipped");
                         break;
@@ -746 +753 @@
                 case TRACE_TYPE_80211_PRISM:
                 case TRACE_TYPE_80211_RADIO:
+                case TRACE_TYPE_ETSILI:
                         assert(!"Metadata headers should already be skipped");
                         break;

lib/protocols_pktmeta.c

@@ -28 +28 @@
 #include "protocols.h"
 #include <assert.h>
+
+#ifdef HAVE_WANDDER
+#include <libwandder_etsili.h>
+#endif
 
 /* This file contains all the protocol decoding functions for the meta-data
@@ -121 +125 @@
 }
 
+static void *trace_get_payload_from_etsili(const void *link,
+                libtrace_linktype_t *type, uint32_t *remaining) {
+
+#ifdef HAVE_WANDDER
+        wandder_decoder_t dec;
+        uint8_t *ccptr;
+
+        init_wandder_decoder(&dec, (uint8_t *)link, *remaining, false);
+        ccptr = wandder_etsili_get_cc_contents(&dec, remaining);
+        /* Assuming all CCs are IP for now */
+        *type = TRACE_TYPE_NONE;
+        return ccptr;
+
+#else
+        *remaining = NULL;
+        return NULL;
+#endif
+
+}
+
 DLLEXPORT void *trace_get_packet_meta(const libtrace_packet_t *packet, 
                 libtrace_linktype_t *linktype,
@@ -139 +163 @@
                 case TRACE_TYPE_80211_PRISM:
                 case TRACE_TYPE_ERF_META:
+                case TRACE_TYPE_ETSILI:
                         return pktbuf;
                 /* Non metadata packets */
@@ -199 +224 @@
                                         linktype, remaining);
                         return nexthdr;
+                case TRACE_TYPE_ETSILI:
+                        nexthdr = trace_get_payload_from_etsili(meta,
+                                        linktype, remaining);
+                        return nexthdr;
+
                 case TRACE_TYPE_HDLC_POS:
                 case TRACE_TYPE_ETH:

lib/trace.c

@@ -148 +148 @@
                 rt_constructor();
                 ndag_constructor();
+#ifdef HAVE_WANDDER
                 etsilive_constructor();
+#endif
 #ifdef HAVE_DAG
                 dag_constructor();

libpacketdump/Makefile.am

@@ -60 +60 @@
 # 17: PPP
 TXT_PROTOCOLS+=link_17.protocol
+
+# 21: ETSI LI
+BIN_PROTOCOLS+=link_21.la
 
 # Decoders for various ethertypes (in decimal)
@@ -135 +138 @@
 link_11_la_LDFLAGS=$(modflags)
 link_15_la_LDFLAGS=$(modflags) 
+link_21_la_LDFLAGS=$(modflags) 
 eth_0_la_LDFLAGS=$(modflags)
 eth_2048_la_LDFLAGS=$(modflags)