Merge pull request meta-toolkit#1 from meta-toolkit/master

update

.gitignore

@@ -15,3 +15,6 @@ tester
 .*
 data/ceeaus
 data/breast-cancer
+biicode.conf
+bii/
+bin/

.travis.yml

@@ -35,7 +35,7 @@ install:
   - if [ "`echo $CXX`" == "clang++" ]; then cd libcxx; fi
   - if [ "`echo $CXX`" == "clang++" ]; then mkdir build; fi
   - if [ "`echo $CXX`" == "clang++" ]; then cd build; fi
-  - if [ "`echo $CXX`" == "clang++" ]; then cmake -DLIBCXX_CXX_ABI=libcxxrt -DLIBCXX_LIBCXXRT_INCLUDE_PATHS="../../libcxxrt/src" -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr ..; fi
+  - if [ "`echo $CXX`" == "clang++" ]; then cmake -DLIBCXX_CXX_ABI=libcxxrt -DLIBCXX_CXX_ABI_INCLUDE_PATHS="../../libcxxrt/src" -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr ..; fi
   - if [ "`echo $CXX`" == "clang++" ]; then make; fi
   - if [ "`echo $CXX`" == "clang++" ]; then sudo make install; fi
   - if [ "`echo $CXX`" == "clang++" ]; then cd $cwd; fi

CHANGELOG.md

@@ -1,5 +1,43 @@
-# [Unreleased][unreleased]
-- nothing yet!
+# [v1.3.5][1.3.5]
+## Bug fixes
+- Fix missing support for sequence/parser analyzers in the classify tools
+
+# [v1.3.4][1.3.4]
+## New features
+- Support building with biicode
+- Add Vagrantfile for virtual machine configuration
+- Add Dockerfile for Docker support
+
+## Enhancements
+- Improve `ir_eval` unit tests
+
+## Bug fixes
+- Fix `ir_eval::ndcg` incorrect log base and addition instead of subtraction in
+    IDCG calculation
+- Fix `ir_eval::avg_p` incorrect early termination
+
+# [v1.3.3][1.3.3]
+## Bug fixes
+- Fix issues with system-defined integer widths in binary model files
+    (mainly impacted the greedy tagger and parser); please re-download any
+    parser model files you may have had before
+- Fix bug where parser model directory is not created if a non-standard
+    prefix is used (anything other than "parser")
+
+## Enhancements
+- Silence inconsistent missing overrides warning on clang >= 3.6
+
+# [v1.3.2][1.3.2]
+## Bug fixes
+- fix potentially incorrect generation of vocabulary map files on 32-bit
+    systems (this appears to have only impacted non-default block sizes)
+
+# [v1.3.1][1.3.1]
+## Bug fixes
+- fix calculation of average precision in `ir_eval` (the denominator was
+    incorrect)
+- specify that labels are required for the `file_corpus` document list; this
+    allows spaces in the path to each document
 
 # [v1.3][1.3]
 ## New features
@@ -89,7 +127,12 @@
 # [v1.0][1.0]
 - Initial release.
 
-[unreleased]: https://github.com/meta-toolkit/meta/compare/v1.3...develop
+[unreleased]: https://github.com/meta-toolkit/meta/compare/v1.3.5...develop
+[1.3.5]: https://github.com/meta-toolkit/meta/compare/v1.3.4...v1.3.5
+[1.3.4]: https://github.com/meta-toolkit/meta/compare/v1.3.3...v1.3.4
+[1.3.3]: https://github.com/meta-toolkit/meta/compare/v1.3.2...v1.3.3
+[1.3.2]: https://github.com/meta-toolkit/meta/compare/v1.3.1...v1.3.2
+[1.3.1]: https://github.com/meta-toolkit/meta/compare/v1.3...v1.3.1
 [1.3]: https://github.com/meta-toolkit/meta/compare/v1.2...v1.3
 [1.2]: https://github.com/meta-toolkit/meta/compare/v1.1...v1.2
 [1.1]: https://github.com/meta-toolkit/meta/compare/v1.0...v1.1

CMakeLists.txt

@@ -14,7 +14,6 @@ include(FindZLIB)
 
 include_directories(include/)
 
-
 set(CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/deps/findicu)
 find_package(Threads REQUIRED)
 find_package(ICU COMPONENTS data i18n uc REQUIRED)
@@ -174,6 +173,11 @@ if(ICU_VERSION VERSION_LESS "4.4")
                              -DMETA_ICU_NO_TEMP_SUBSTRING)
 endif()
 
+if(BIICODE)
+  include(contrib/biicode/CMakeLists.txt)
+  return()
+ENDIF()
+
 set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})
 
 link_libraries(cpptoml meta-definitions)

README.md

@@ -404,6 +404,6 @@ system.
    appropriate `cmake` command for your OS above and rebuild using `make`
    after.
 
-[meta-website]: https://meta-toolkit.github.io/meta/
+[meta-website]: https://meta-toolkit.org
 [doxygen]: https://meta-toolkit.github.io/meta/doxygen/namespaces.html
 [homebrew]: http://brew.sh

contrib/YouCompleteMe/README.md

@@ -0,0 +1,11 @@
+# YouCompleteMe Integration
+This is an extra configuration file for use with the excellent
+[YouCompleteMe] plugin for vim.
+
+Do the following from the project's root directory to start using it:
+
+```bash
+ln -s contrib/YouCompleteMe/ycm_extra_conf.py .ycm_extra_conf.py
+```
+
+[YouCompleteMe]: https://github.com/Valloric/YouCompleteMe

contrib/YouCompleteMe/ycm_extra_conf.py

@@ -0,0 +1,95 @@
+# .ycm_extra_conf.py for MeTA source code.
+# Based on the provided ycm_extra_conf from YouCompleteMe as well as the
+# ycm_extra_conf.py from neovim.
+import os
+import ycm_core
+
+def DirectoryOfThisScript():
+  return os.path.dirname( os.path.abspath( __file__ ) )
+
+def GetDatabase():
+  build_folder = os.path.join(DirectoryOfThisScript(), 'build')
+  if (os.path.exists(build_folder)):
+    return ycm_core.CompilationDatabase(build_folder)
+  return None
+
+def MakeRelativePathsInFlagsAbsolute(flags, working_directory):
+  if not working_directory:
+    return flags
+  new_flags = []
+  make_next_absolute = False
+  path_flags = ['-isystem', '-I', '-iquote', '--sysroot=']
+  for flag in flags:
+    new_flag = flag
+
+    if make_next_absolute:
+      make_next_absolute = False
+      if not flag.startswith('/'):
+        new_flag = os.path.join(working_directory, flag)
+
+    for path_flag in path_flags:
+      if flag == path_flag:
+        make_next_absolute = True
+        break
+
+      if flag.startswith(path_flag):
+        path = flag[len(path_flag):]
+        new_flag = path_flag + os.path.join(working_directory, path)
+        break
+
+    if new_flag:
+      new_flags.append(new_flag)
+  return new_flags
+
+def IsHeader(filename):
+  headers = set(['.h', '.hpp', '.tcc'])
+  return os.path.splitext(filename)[1] in headers
+
+def GetCompilationInfoForFile(filename):
+  database = GetDatabase()
+  if not database:
+    return None
+
+  if IsHeader(filename):
+    # CMake doesn't generate flags for header files in its compilation
+    # database, so we attempt to look up the corresponding cpp file in the
+    # source directory and use its flags
+    base_name = os.path.splitext(filename)[0]
+    cpp_name = base_name.replace('/include', '/src') + '.cpp'
+
+    # if we can't find the corresponding cpp, fall back to the flags for
+    # profile.cpp
+    if not os.path.exists(cpp_name):
+      cpp_name = os.path.join(DirectoryOfThisScript(), 'src', 'tools',
+          'profile.cpp')
+
+    if not os.path.exists(cpp_name):
+      return None
+
+    compilation_info = database.GetCompilationInfoForFile(cpp_name)
+    # fixes issues where .h files seemingly can't find c++ standard header
+    # files
+    if compilation_info.compiler_flags_:
+      compilation_info.compiler_flags_.append('-x')
+      compilation_info.compiler_flags_.append('c++')
+    return compilation_info
+
+  if not os.path.exists(filename):
+    return None
+
+  return database.GetCompilationInfoForFile(filename)
+
+
+def FlagsForFile( filename ):
+  compilation_info = GetCompilationInfoForFile(filename)
+  if not compilation_info:
+    return None
+
+  final_flags = MakeRelativePathsInFlagsAbsolute(
+      compilation_info.compiler_flags_,
+      compilation_info.compiler_working_dir_)
+
+  return {
+    'flags': final_flags,
+    'do_cache': True
+  }

contrib/biicode/CMakeLists.txt

@@ -0,0 +1,8 @@
+message(STATUS "Setting up biicode block...")
+set(UNIT_TEST_EXE skystrife_meta_src_test_tools_unit-test)
+include(src/test/unit_tests.cmake)
+ADD_BIICODE_TARGETS()
+target_link_libraries(${BII_LIB_TARGET} PUBLIC meta-definitions
+                                               ${ICU_LIBRARIES}
+                                               ${ZLIB_LIBRARIES}
+                                               ${CMAKE_THREAD_LIB_INIT})

contrib/biicode/README.md

@@ -0,0 +1,22 @@
+# Biicode Integration
+MeTA can be built using biicode. To do so, symlink the biicode.conf file to
+the project's root directory like so (from the project root):
+
+```bash
+ln -s contrib/biicode/biicode.conf .
+```
+
+Then, configure and build with the following:
+
+```bash
+bii init -L
+bii configure
+bii build
+```
+
+After placing the sample config.toml in the `bin/` directory, you can run
+the unit tests:
+
+```bash
+bii test
+```

contrib/biicode/biicode.conf

@@ -0,0 +1,60 @@
+# Biicode configuration file
+
+[requirements]
+    skystrife/cpptoml: 0
+
+[parent]
+    # The parent version of this block. Must match folder name. E.g.
+    # user/block  # No version number means not published yet
+    # You can change it to publish to a different track, and change version, e.g.
+    # user/block(track): 7
+    skystrife/meta: 0
+
+[paths]
+    # Local directories to look for headers (within block)
+    # /
+    include
+    # should put porter2_stemmer on biicode as well...
+    deps/porter2_stemmer
+
+[dependencies]
+    # Manual adjust file implicit dependencies, add (+), remove (-), or overwrite (=)
+    # hello.h + hello_imp.cpp hello_imp2.cpp
+    # *.h + *.cpp
+    include/analyzers/* + src/analyzers/* src/corpus/*
+    include/classify/* + src/classify/* src/index/*
+    include/corpus/* + src/corpus/* src/io/*
+    include/index/* + src/index/* src/analyzers/*
+    include/io/* + src/io/* src/util/*
+    include/lm/* + src/lm/* src/corpus/*
+    include/parser/* + src/parser/* src/io/* src/sequence/* src/util/*
+    include/sequence/* + src/sequence/* src/io/* src/utf/*
+    include/topics/* + src/topics/* src/index/*
+    CMakeLists.txt + deps/findicu/FindICU.cmake src/test/unit_tests.cmake contrib/biicode/CMakeLists.txt
+
+[mains]
+    # Manual adjust of files that define an executable
+    # !main.cpp  # Do not build executable from this file
+    # main2.cpp # Build it (it doesnt have a main() function, but maybe it includes it)
+    !deps/*
+
+[tests]
+    # Manual adjust of files that define a CTest test
+    # test/* pattern to evaluate this test/ folder sources like tests
+
+[hooks]
+    # These are defined equal to [dependencies],files names matching bii*stage*hook.py
+    # will be launched as python scripts at stage = {post_process, clean}
+    # CMakeLists.txt + bii/my_post_process1_hook.py bii_clean_hook.py
+
+[includes]
+    # Mapping of include patterns to external blocks
+    # hello*.h: user3/depblock  # includes will be processed as user3/depblock/hello*.h
+    cpptoml.h: skystrife/cpptoml/include
+
+[data]
+    # Manually define data files dependencies, that will be copied to bin for execution
+    # By default they are copied to bin/user/block/... which should be taken into account
+    # when loading from disk such data
+    # image.cpp + image.jpg  # code should write open("user/block/image.jpg")
+

contrib/docker/Dockerfile

@@ -0,0 +1,20 @@
+FROM debian:jessie
+
+MAINTAINER Maciej Szymkiewicz "[email protected]"
+
+ENV METADIR /opt/meta
+RUN mkdir -p $METADIR
+
+RUN apt-get update && apt-get -y install cmake libicu-dev git g++ && apt-get clean
+
+WORKDIR $METADIR
+RUN git clone --depth 1 https://github.com/meta-toolkit/meta.git .
+RUN git submodule update --init --recursive
+RUN mkdir $METADIR/build
+
+WORKDIR $METADIR/build
+RUN cp  $METADIR/config.toml .
+RUN cmake $METADIR -DCMAKE_BUILD_TYPE=Release && make
+RUN ctest --output-on-failure
+
+RUN apt-get -y purge git && apt-get -y autoremove 

contrib/vagrant/README.md

@@ -0,0 +1,22 @@
+# Vagrant
+
+Build the latest version of the MeTA toolkit using [Vagrant](https://www.vagrantup.com/). Ubuntu 14.04 is used as a base Vagrant box. This project was initially made for [Text Retrieval and Search Engines](https://www.coursera.org/course/textretrieval) Coursera course.
+
+## Instructions
+
+1. Install Vagrant  
+2. Run Vagrant
+    ```
+    vagrant up
+    ```
+    
+3. Check that everything is OK
+  ```
+    vagrant ssh
+    cd meta/build
+    ctest --output-on-failure
+    ```
+
+## Notes
+
+Only VirtualBox provider was tested. 

contrib/vagrant/Vagrantfile

@@ -0,0 +1,11 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+Vagrant.configure(2) do |config|
+  config.vm.box = "ubuntu/trusty64"
+  config.vm.provision :shell, path: "bootstrap.sh", privileged: false
+
+  config.vm.provider "virtualbox" do |v|
+    v.cpus = 2
+  end
+end

contrib/vagrant/bootstrap.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+
+# this might take a while
+sudo apt-get update
+sudo apt-get install -y software-properties-common
+
+# add the ppa for cmake
+sudo add-apt-repository -y ppa:george-edison55/cmake-3.x
+sudo apt-get update
+
+# install dependencies
+sudo apt-get install -y cmake libicu-dev git g++ g++-4.8
+
+# clone the project
+git clone https://github.com/meta-toolkit/meta.git
+cd meta/
+
+# uncomment to build exact version
+# git reset --hard v1.3.2
+
+# set up submodules
+git submodule update --init --recursive
+
+# set up a build directory
+mkdir build
+cd build
+cp ../config.toml .
+
+# configure and build the project
+cmake ../ -DCMAKE_BUILD_TYPE=Release
+make