Master hotfix 1.6.1 (#230)

* update

* removed submodules from master

* emergency release of version 1.6.1, with all submodules flattened. It should compile on all platforms.

* removed macos from git actions. If you want macos, you will need to install openssl through macports. We have support for the native mac crypto libraries in the next version.

* removed the darwin stuff that was causing linux configure.ac to crash

* just ubuntu 20.04 now

* install more of boost

* more typos

Author: simsong

.github/workflows/continuous-integration.yml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: ['ubuntu-18.04', 'ubuntu-20.04', 'macos-10.15']
+        os: ['ubuntu-20.04']
 
     steps:
       # https://github.com/actions/checkout
@@ -29,11 +29,10 @@ jobs:
       - name: Install C++ dependencies on Ubuntu
         if: startsWith(matrix.os, 'ubuntu')
         run: |
-           sudo apt install libtool autoconf automake g++ libboost-dev libssl-dev libpcap-dev libcairo2-dev
+           sudo apt install libtool autoconf automake g++ libboost-all-dev libssl-dev libpcap-dev libcairo2-dev
            automake --help
 
       - name: c/c++ test
         run: |
           bash bootstrap.sh
-          ./configure && make && make check 
-
+          ./configure && make && make check

.gitignore

@@ -93,3 +93,4 @@ tests/*.pdf
 tests/out
 tests/packet1.pcap
 *.d
+ar-lib

.gitmodules

@@ -1,11 +1,3 @@
-[submodule "src/be13_api"]
-	path = src/be13_api
-	url = https://github.com/simsong/be13_api.git
-	branch = master
-[submodule "src/dfxml"]
-	path = src/dfxml_x
-	url = https://github.com/simsong/dfxml.git
-	branch = master
 [submodule "src/http-parser"]
 	path = src/http-parser
 	url = https://github.com/simsong/http-parser.git

configure.ac

@@ -7,7 +7,7 @@
 # and http://www.openismus.com/documents/linux/automake/automake.shtml
 
 AC_PREREQ(2.57)
-AC_INIT(TCPFLOW, 1.6.0, [email protected])
+AC_INIT(TCPFLOW, 1.6.1, [email protected])
 AC_CONFIG_MACRO_DIR([m4])
 
 AC_CONFIG_FILES([Makefile src/Makefile tests/Makefile doc/Makefile])
@@ -143,50 +143,7 @@ done
 unset option
 
 
-# C++
-# We don't use these warnings:
-# -Waggregate-return -- aggregate returns are GOOD; they simplify code design
-# We can use these warnings after ZLIB gets upgraded:
-# -Wundef  --- causes problems with zlib
-# -Wcast-qual
-# -Wmissing-format-attribute  --- Just too annoying
-AC_LANG_PUSH(C++)
-WARNINGS_TO_TEST="-Wall -MD -D_FORTIFY_SOURCE=2 -Wpointer-arith \
-    -Wshadow -Wwrite-strings -Wcast-align  \
-    -Wredundant-decls -Wdisabled-optimization \
-    -Wfloat-equal -Wmultichar -Wmissing-noreturn \
-    -Woverloaded-virtual -Wsign-promo \
-    -funit-at-a-time"
-
-add_strict_null_sentinel='yes'
-if $CXX -dM -E - < /dev/null | grep -q __clang__ ; then
-   AC_MSG_RESULT([Using clang])
-   add_strict_null_sentinel='no'
-fi
-
-if uname -a | grep -q Darwin; then
-   add_strict_null_sentinel='no'
-fi
-
-if [ $add_strict_null_sentinel = 'yes' ]; then
-   WARNINGS_TO_TEST="$WARNINGS_TO_TEST -Wstrict-null-sentinel"
-fi
-
-for option in $WARNINGS_TO_TEST
-do
-  SAVE_CXXFLAGS="$CXXFLAGS"
-  CXXFLAGS="$CXXFLAGS $option"
-  AC_MSG_CHECKING([whether g++ understands $option])
-  AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[]], [[]])],
-      [has_option=yes],
-      [has_option=no; CXXFLAGS="$SAVE_CXXFLAGS"])
-  AC_MSG_RESULT($has_option)
-  unset has_option
-  unset SAVE_CXXFLAGS
-done
-unset option
-AC_LANG_POP()
-
+# CUT1
 ################################################################
 ##
 

src/Makefile.am

@@ -104,7 +104,7 @@ tcpflow_SOURCES = \
 	mime_map.h
 
 # Removed because it hasn't been updated to Python 3:
-	scan_python.cpp \
+#	scan_python.cpp
 
 
 EXTRA_DIST =\

src/be13_api

@@ -0,0 +1,36 @@
+BE13_API= \
+        be13_api/atomic_set_map.h \
+        be13_api/aftimer.h \
+        be13_api/beregex.cpp \
+        be13_api/beregex.h \
+        be13_api/bulk_extractor_i.h \
+        be13_api/cppmutex.h \
+        be13_api/feature_recorder.cpp \
+        be13_api/feature_recorder.h \
+        be13_api/feature_recorder_set.cpp \
+        be13_api/feature_recorder_set.h \
+        be13_api/feature_recorder_sql.cpp \
+        be13_api/histogram.h \
+        be13_api/histogram.cpp \
+        be13_api/net_ethernet.h \
+        be13_api/pcap_fake.cpp \
+        be13_api/pcap_fake.h \
+        be13_api/plugin.cpp \
+        be13_api/sbuf.cpp \
+        be13_api/sbuf.h \
+        be13_api/sbuf_private.h \
+        be13_api/sbuf_stream.cpp \
+        be13_api/sbuf_stream.h \
+        be13_api/unicode_escape.cpp \
+        be13_api/unicode_escape.h \
+        be13_api/utf8.h \
+        be13_api/utf8/checked.h \
+        be13_api/utf8/core.h \
+        be13_api/utf8/unchecked.h \
+        be13_api/utils.cpp \
+        be13_api/utils.h \
+        be13_api/word_and_context_list.cpp \
+        be13_api/word_and_context_list.h 
+
+
+

src/be13_api/Makefile.defs

@@ -0,0 +1,178 @@
+#ifndef __AFTIMER_H__
+#define __AFTIMER_H__
+
+#ifdef __cplusplus
+#ifndef WIN32
+#ifndef __STDC_FORMAT_MACROS
+#define __STDC_FORMAT_MACROS
+#endif
+#include <inttypes.h>
+#include <sys/time.h>
+#endif
+#include <sys/types.h>
+#include <stdio.h>
+#include <string>
+
+class aftimer {
+    struct timeval t0;
+    bool running;
+    long total_sec;
+    long total_usec;
+    double lap_time_;			// time from when we last did a "stop"
+public:
+    aftimer():t0(),running(false),total_sec(0),total_usec(0),lap_time_(0){}
+
+    void start();			// start the timer
+    void stop();			// stop the timer
+
+    time_t tstart() const { return t0.tv_sec;} // time we started
+    double elapsed_seconds() const;	 // how long timer has been running, total
+    double lap_time() const;		 // how long the timer is running this time
+    double eta(double fraction_done) const;	// calculate ETA in seconds, given fraction
+    std::string hms(long t) const;	// turn a number of seconds into h:m:s
+    std::string elapsed_text() const;		/* how long we have been running */
+    std::string eta_text(double fraction_done) const; // h:m:s
+    std::string eta_time(double fraction_done) const; // the actual time
+};
+
+/* This code in part from 
+ * http://social.msdn.microsoft.com/Forums/en/vcgeneral/thread/430449b3-f6dd-4e18-84de-eebd26a8d668
+ */
+
+#if defined(WIN32) || defined(__MINGW32__)
+#  include <winsock2.h>
+#  include <windows.h>			
+#  ifndef DELTA_EPOCH_IN_MICROSECS
+#    if defined(_MSC_VER) || defined(_MSC_EXTENSIONS)
+#      define DELTA_EPOCH_IN_MICROSECS  11644473600000000Ui64
+#    else
+#      define DELTA_EPOCH_IN_MICROSECS  11644473600000000ULL
+#    endif
+#  endif
+#endif
+
+inline void timestamp(struct timeval *t)
+{
+#ifdef WIN32
+    FILETIME ft;
+    GetSystemTimeAsFileTime(&ft);
+    unsigned __int64 tmpres = 0;
+    tmpres |= ft.dwHighDateTime;
+    tmpres <<= 32;
+    tmpres |= ft.dwLowDateTime;
+ 
+    /*converting file time to unix epoch*/
+    tmpres -= DELTA_EPOCH_IN_MICROSECS; 
+    tmpres /= 10;  /*convert into microseconds*/
+    t->tv_sec = (long)(tmpres / 1000000UL);
+    t->tv_usec = (long)(tmpres % 1000000UL);
+#else
+    gettimeofday(t,NULL);
+#endif    
+}
+
+inline void aftimer::start()
+{
+    timestamp(&t0);
+    running = 1;
+}
+
+inline void aftimer::stop(){
+    if(running){
+	struct timeval t;
+	timestamp(&t);
+	total_sec  += t.tv_sec - t0.tv_sec;
+	total_usec += t.tv_usec - t0.tv_usec;
+	lap_time_   = (double)(t.tv_sec - t0.tv_sec)  + (double)(t.tv_usec - t0.tv_usec)/1000000.0;
+	running = false;
+    }
+}
+
+inline double aftimer::lap_time() const
+{
+    return lap_time_;
+}
+
+inline double aftimer::elapsed_seconds() const
+{
+    double ret = (double)total_sec + (double)total_usec/1000000.0;
+    if(running){
+	struct timeval t;
+	timestamp(&t);
+	ret += t.tv_sec - t0.tv_sec;
+	ret += (t.tv_usec - t0.tv_usec) / 1000000.0;
+    }
+    return ret;
+}
+
+inline std::string aftimer::hms(long t) const
+{
+    char   buf[64];
+    int    days = t / (60*60*24);
+    
+    t = t % (60*60*24);			/* what's left */
+
+    int    h = t / 3600;
+    int    m = (t / 60) % 60;
+    int    s = t % 60;
+    buf[0] = 0;
+    switch(days){
+    case 0:
+	snprintf(buf,sizeof(buf),"%2d:%02d:%02d",h,m,s);
+	break;
+    case 1:
+	snprintf(buf,sizeof(buf),"%d day, %2d:%02d:%02d",days,h,m,s);
+	break;
+    default:
+	snprintf(buf,sizeof(buf),"%d days %2d:%02d:%02d",days,h,m,s);
+    }
+    return std::string(buf);
+}
+
+inline std::string aftimer::elapsed_text() const
+{
+    return hms((int)elapsed_seconds());
+}
+
+/**
+ * returns the number of seconds until the job is complete.
+ */
+inline double aftimer::eta(double fraction_done) const
+{
+    double t = elapsed_seconds();
+    if(t<=0) return -1;			// can't figure it out
+    if(fraction_done<=0) return -1;	// can't figure it out
+    return (t * 1.0/fraction_done - t);
+}
+
+/**
+ * Retuns the number of hours:minutes:seconds until the job is done.
+ */
+inline std::string aftimer::eta_text(double fraction_done) const
+{
+    double e = eta(fraction_done);
+    if(e<0) return std::string("n/a");		// can't figure it out
+    return hms((long)e);
+}
+
+/**
+ * Returns the time when data is due.
+ */
+inline std::string aftimer::eta_time(double fraction_done) const 
+{
+    time_t t = time_t(eta(fraction_done)) + time(0);
+    struct tm tm;
+#ifdef HAVE_LOCALTIME_R
+    localtime_r(&t,&tm);
+#else
+    tm = *localtime(&t);
+#endif
+    
+    char buf[64];
+    snprintf(buf,sizeof(buf),"%02d:%02d:%02d",tm.tm_hour,tm.tm_min,tm.tm_sec);
+    return std::string(buf);
+}
+
+#endif
+
+#endif