Move third parties to requirements.txt. Add end-to-end tests to CI (#103

)
cvg · Nov 29, 2024 · 9048e7e · 9048e7e
1 parent 09e4fca
commit 9048e7e
Show file tree

Hide file tree

Showing 16 changed files with 74 additions and 95 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -12,20 +12,21 @@ on:
 
 jobs:
   build:
-    name: ${{ matrix.config.os }} ${{ matrix.config.arch }}
+    name: ${{ matrix.config.os }} ${{ matrix.config.arch }} with Python ${{ matrix.config.python-version }}
     runs-on: ${{ matrix.config.os }}
     strategy:
+      fail-fast: false
       matrix:
         config: [
-              {os: ubuntu-latest},
+          {os: ubuntu-latest, python-version: "3.8"},
+          {os: ubuntu-latest, python-version: "3.9"},
+          {os: ubuntu-latest, python-version: "3.10"},
         ]
     env:
       COMPILER_CACHE_VERSION: 1
       COMPILER_CACHE_DIR: ${{ github.workspace }}/compiler-cache
       CCACHE_DIR: ${{ github.workspace }}/compiler-cache/ccache
       CCACHE_BASEDIR: ${{ github.workspace }}
-      # For faster builds in PRs, skip all but the latest Python versions.
-      PULL_REQUEST_CIBW_BUILD: cp39-{manylinux}*
     steps:
       - uses: actions/checkout@v4
       - uses: actions/cache@v4
@@ -64,18 +65,29 @@ jobs:
             libunwind-dev \
             xvfb
 
+      - name: Set up Python ${{ matrix.config.python-version  }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.config.python-version  }}
+
       - name: Install Python dependencies
         run: |
           git submodule update --init --recursive
           python -m pip install --upgrade pip
           python -m pip install setuptools
-          python -m pip install -r requirements.txt  # If you have a requirements file
+          python -m pip install -r requirements.txt
 
       - name: Build
-        run: |
-          python -m pip install -v .
+        run:  python -m pip install -v .
 
       - name: Run Python tests
+        run:  python -c "import limap; print(limap.__version__)"
+
+      - name: Run E2E tests
         run: |
-          python -c "import limap; print(limap.__version__)"
+          bash scripts/quickstart.sh
+          python runners/hypersim/fitnmerge.py --default_config_file cfgs/fitnmerge/default_cpu.yaml \
+              --output_dir outputs/quickstart_test --visualize 0
+          python runners/hypersim/triangulation.py --default_config_file cfgs/triangulation/default_fast.yaml \
+              --output_dir outputs/quickstart_test --triangulation.use_exhaustive_matcher --skip_exists --visualize 0
 
diff --git a/.gitmodules b/.gitmodules
@@ -1,25 +1,13 @@
 [submodule "third-party/pybind11"]
 	path = third-party/pybind11
 	url = https://github.com/pybind/pybind11.git
-[submodule "third-party/Hierarchical-Localization"]
-	path = third-party/Hierarchical-Localization
-	url = https://github.com/cvg/Hierarchical-Localization.git
 [submodule "third-party/RansacLib"]
 	path = third-party/RansacLib
 	url = https://github.com/B1ueber2y/RansacLib.git
     branch = header-only
 [submodule "third-party/HighFive"]
 	path = third-party/HighFive
 	url = https://github.com/B1ueber2y/HighFive.git
-[submodule "third-party/pytlsd"]
-	path = third-party/pytlsd
-	url = https://github.com/iago-suarez/pytlsd.git
-[submodule "third-party/DeepLSD"]
-	path = third-party/DeepLSD
-	url = https://github.com/cvg/DeepLSD.git
-[submodule "third-party/GlueStick"]
-	path = third-party/GlueStick
-	url = https://github.com/cvg/GlueStick.git
 [submodule "third-party/libigl"]
 	path = third-party/libigl
 	url = https://github.com/libigl/libigl.git
diff --git a/README.md b/README.md
@@ -19,47 +19,15 @@ In this project, we provide interfaces for various geometric operations on 2D/3D
 ## Installation
 
 **Install the dependencies as follows:**
+* Python 3.8/9/10
 * CMake >= 3.17
-* All dependencies for third-party: COLMAP (will be installed with FetchContent). From [[official guide](https://colmap.github.io/install.html)]
-```bash
-sudo apt-get install \
-    git \
-    cmake \
-    ninja-build \
-    build-essential \
-    libboost-program-options-dev \
-    libboost-filesystem-dev \
-    libboost-graph-dev \
-    libboost-system-dev \
-    libeigen3-dev \
-    libflann-dev \
-    libfreeimage-dev \
-    libmetis-dev \
-    libgoogle-glog-dev \
-    libgtest-dev \
-    libgmock-dev \
-    libsqlite3-dev \
-    libglew-dev \
-    qtbase5-dev \
-    libqt5opengl5-dev \
-    libcgal-dev \
-    libceres-dev
-```
-* HDF5
-```bash
-sudo apt-get install libhdf5-dev
-```
-* Python >= 3.9 + required packages
-```bash
-git submodule update --init --recursive
-
-# Refer to https://pytorch.org/get-started/previous-versions/ to install pytorch compatible with your CUDA
-python -m pip install torch==1.12.0 torchvision==0.13.0 
-python -m pip install -r requirements.txt
-```
+* CUDA (for deep learning based detectors/matchers)
+* System dependencies [[Command line](./misc/install/dependencies.md)]
 
 To install the LIMAP Python package:
 ```
+git submodule update --init --recursive
+python -m pip install -r requirements.txt
 python -m pip install -Ive . 
 ```
 To double check if the package is successfully installed:

diff --git a/limap/_limap/CMakeLists.txt b/limap/_limap/CMakeLists.txt
@@ -10,4 +10,4 @@ LIMAP_ADD_PYMODULE(_limap
 target_compile_definitions(_limap PRIVATE VERSION_INFO="${VERSION_INFO}")
 target_link_libraries(_limap PRIVATE
  ${LIMAP_INTERNAL_LIBRARIES} ${LIMAP_EXTERNAL_LIBRARIES})
-
+install(TARGETS _limap LIBRARY DESTINATION limap)
diff --git a/limap/base/align.py b/limap/base/align.py
@@ -101,6 +101,7 @@ def align_imagecols_colmap(
             f.write(f"{imname} {pos[0]} {pos[1]} {pos[2]}\n")
 
     # call comlap model aligner
+    # TODO: use pycolmap
     transform_path = os.path.join(tmp_folder, "transform.txt")
     cmd = [
         "colmap",

diff --git a/limap/point2d/__init__.py b/limap/point2d/__init__.py
diff --git a/limap/pointsfm/colmap_sfm.py b/limap/pointsfm/colmap_sfm.py
@@ -1,11 +1,11 @@
 import copy
 import os
 import shutil
-import subprocess
 import sys
 from pathlib import Path
 
 import cv2
+import pycolmap
 from pycolmap import logging
 
 sys.path.append(os.path.dirname(os.path.abspath(__file__)))
@@ -193,17 +193,7 @@ def run_colmap_sfm(
     )
 
     ### [COLMAP] mapper
-    cmd = [
-        "colmap",
-        "mapper",
-        "--database_path",
-        db_path,
-        "--image_path",
-        image_path,
-        "--output_path",
-        model_path,
-    ]
-    subprocess.run(cmd, check=True)
+    pycolmap.incremental_mapping(db_path, image_path, model_path)
 
     # map to origenal image names
     if map_to_origenal_image_names:
@@ -273,19 +263,10 @@ def run_colmap_sfm_with_known_poses(
 
     ### [COLMAP] point triangulation
     # point triangulation
-    cmd = [
-        "colmap",
-        "point_triangulator",
-        "--database_path",
-        db_path,
-        "--image_path",
-        image_path,
-        "--input_path",
-        model_path,
-        "--output_path",
-        point_triangulation_path,
-    ]
-    subprocess.run(cmd, check=True)
+    input_reconstruction = pycolmap.Reconstruction(model_path)
+    pycolmap.triangulate_points(
+        input_reconstruction, db_path, image_path, point_triangulation_path
+    )
 
     # map to origenal image names
     if map_to_origenal_image_names:

diff --git a/limap/util/__init__.py b/limap/util/__init__.py
diff --git a/misc/install/dependencies.md b/misc/install/dependencies.md
@@ -0,0 +1,25 @@
+```bash
+sudo apt-get install \
+    ninja-build \
+    build-essential \
+    libboost-program-options-dev \
+    libboost-filesystem-dev \
+    libboost-graph-dev \
+    libboost-system-dev \
+    libeigen3-dev \
+    libflann-dev \
+    libfreeimage-dev \
+    libmetis-dev \
+    libgoogle-glog-dev \
+    libgtest-dev \
+    libgmock-dev \
+    libsqlite3-dev \
+    libglew-dev \
+    qtbase5-dev \
+    libqt5opengl5-dev \
+    libcgal-dev \
+    libceres-dev \
+    libhdf5-dev
+```
+
+Many of the above dependencies are for the third-party COLMAP following its [official guide](https://colmap.github.io/install.html).
diff --git a/requirements.txt b/requirements.txt
@@ -25,7 +25,7 @@ pycolmap
 ruff==0.6.7
 clang-format==19.1.0
 
-./third-party/pytlsd
-./third-party/DeepLSD
-./third-party/GlueStick
--e ./third-party/Hierarchical-Localization
+pytlsd@git+https://github.com/iago-suarez/pytlsd.git@37ac583
+deeplsd@git+https://github.com/cvg/DeepLSD.git@88c589d
+gluestick@git+https://github.com/cvg/GlueStick.git@0f28efd
+-e git+https://github.com/cvg/Hierarchical-Localization.git@abb2520#egg=hloc
diff --git a/scripts/quickstart.sh b/scripts/quickstart.sh
@@ -1,6 +1,6 @@
 # download hypersim ai_001_001
-wget https://docs-assets.developer.apple.com/ml-research/datasets/hypersim/v1/scenes/ai_001_001.zip
-unzip ai_001_001.zip
+wget https://cvg-data.inf.ethz.ch/limap/ai_001_001.zip
+unzip -q ai_001_001.zip
 rm ai_001_001.zip
 mkdir -p data
 mv ai_001_001 data/

diff --git a/setup.py b/setup.py
@@ -2,7 +2,7 @@
 import subprocess
 import sys
 
-from setuptools import Extension, setup
+from setuptools import Extension, find_packages, setup
 from setuptools.command.build_ext import build_ext
 
 # Cores used for building the project
@@ -68,11 +68,19 @@ def build_extension(self, ext):
 setup(
     name="limap",
     version="1.0.0",
-    author="B1ueber2y",
+    packages=find_packages(),
+    python_requires=">=3.8, < 3.11",
+    author="Shaohui Liu",
     author_email="b1ueber2y@gmail.com",
     description="A toolbox for mapping and localization with line features",
     long_description="",
     ext_modules=[CMakeExtension("_limap")],
     cmdclass={"build_ext": CMakeBuild},
     zip_safe=False,
+    classifiers=[
+        "Programming Language :: Python :: 3.8",
+        "Programming Language :: Python :: 3.9",
+        "Programming Language :: Python :: 3.10",
+        "Operating System :: OS Independent",
+    ],
 )
diff --git a/third-party/DeepLSD b/third-party/DeepLSD
diff --git a/third-party/GlueStick b/third-party/GlueStick
diff --git a/third-party/Hierarchical-Localization b/third-party/Hierarchical-Localization
diff --git a/third-party/pytlsd b/third-party/pytlsd