Automated build and datadir init

kelvich · kelvich · commit 8d1588b35d91 · 2022-01-04T20:34:03.000+03:00
diff --git a/README_WASM.md b/README_WASM.md
@@ -3,94 +3,35 @@
 
 ## Compilation
 
-export EMMAKEN_CFLAGS="-Wl,--allow-undefined"
-emconfigure ./configure CFLAGS='-Oz' --without-readline --without-zlib --disable-thread-safety --disable-spinlocks
-export EMMAKEN_CFLAGS="-s ERROR_ON_UNDEFINED_SYMBOLS=0 -s TOTAL_MEMORY=65536000 -s EMULATE_FUNCTION_POINTER_CASTS=1"
-emmake make -j4
+Prerequisites:
+* emscripten/3.1.0
+* nodejs
 
-## Setup the data directory
 
-emcc src/bin/initdb/initdb.o -L src/port/ -L src/common -L src/fe_utils -L src/interfaces/libpq -lpq -lpgfeutils -lpgcommon -lpgport -o initdb.js
+To build postgres.wasm and datadir bundle:
+```
+make -f wasm/Makefile debug-build
+make -f wasm/Makefile debug-datadir
+```
 
-# create empty datadir
-rm -rf datadir
-mkdir datadir
-chmod 0750 datadir
-touch datadir/postgresql.conf
-touch datadir/postgresql.auto.conf
-touch datadir/pg_ident.conf
-touch datadir/pg_hba.conf
-echo '15devel' > datadir/PG_VERSION
-mkdir -p datadir/global
-mkdir -p datadir/pg_wal/archive_status
-mkdir -p datadir/pg_commit_ts
-mkdir -p datadir/pg_dynshmem
-mkdir -p datadir/pg_notify
-mkdir -p datadir/pg_serial
-mkdir -p datadir/pg_snapshots
-mkdir -p datadir/pg_subtrans
-mkdir -p datadir/pg_twophase
-mkdir -p datadir/pg_multixact
-mkdir -p datadir/pg_multixact/members
-mkdir -p datadir/pg_multixact/offsets
-mkdir -p datadir/base
-mkdir -p datadir/base/1
-echo '15devel' > datadir/base/1/PG_VERSION
-mkdir -p datadir/pg_replslot
-mkdir -p datadir/pg_tblspc
-mkdir -p datadir/pg_stat
-mkdir -p datadir/pg_stat_tmp
-mkdir -p datadir/pg_xact
-mkdir -p datadir/pg_logical
-mkdir -p datadir/pg_logical/snapshots
-mkdir -p datadir/pg_logical/mapping
+You can check it in the browser. Wasm file would not be loaded as 'file://' links, so you need to have an http server on localhost, e.g.:
+```
+cd wasm
+python3 -m http.server
+```
 
+Now navigate to `http://localhost:8000/`
 
-# add that keys to postgres.o compilation:
---preload-file /Users/stas/datadir --preload-file /usr/local/pgsql/share/
 
-cd src/backend
+## TODO
 
-node postgres --boot -x1 -X 16777216 -d 5 -c dynamic_shared_memory_type=mmap -D /data
+- [x] separate fs packaging from compiling
+- [x] read from events
+- [x] somehow report back results
+- [x] identify and fix FUNCTION_POINTER_CASTS
+- [x] identify and fix corrupt memory error
 
-
-
-node postgres --single -F -O -j -c search_path=pg_catalog -c dynamic_shared_memory_type=mmap -d 5 -D /data template1
-
-
-
-
-------
-
-
-var Module = {
-  preRun: [function() {
-    function stdin() {
-      // Return ASCII code of character, or null if no input
-    }
-
-    function stdout(asciiCode) {
-      // Do something with the asciiCode
-    }
-
-    function stderr(asciiCode) {
-      // Do something with the asciiCode
-    }
-
-    FS.init(stdin, stdout, stderr);
-  }]
-};
-
-
-# todo
-
-+ separate fs packaging from compiling
-+ read from events
-+ somehow report back
-+ identify and fix FUNCTION_POINTER_CASTS
-+ fix report back corrupt memory
-
-* automate datadir process
+- [x] automate datadir process
 * split to commits, ensure usual postgres is properly built
 * provide js wrapper
 * nice UI
@@ -103,4 +44,6 @@ var Module = {
 
 * regress test? =)
 
+* "runnable" sql embeds
+
 * work with the remote data
diff --git a/src/backend/Makefile b/src/backend/Makefile
@@ -63,11 +63,7 @@ ifneq ($(PORTNAME), win32)
 ifneq ($(PORTNAME), aix)
 
 postgres: $(OBJS)
-	$(CC) $(CFLAGS) $(call expand_subsys,$^) $(LDFLAGS) $(LDFLAGS_EX) $(export_dynamic) $(LIBS) -s FORCE_FILESYSTEM=1 -s ASYNCIFY=1 -o $@.html
-
-# --preload-file /Users/stas/datadir --preload-file /usr/local/pgsql/share -o $@.html
-# --embed-file /Users/stas/datadir --embed-file /usr/local/pgsql/share -lnodefs.js -ltty.js
-# --embed-file /Users/stas/datadir/
+	$(CC) $(CFLAGS) $(call expand_subsys,$^) $(LDFLAGS) $(LDFLAGS_EX) $(export_dynamic) $(LIBS) -s FORCE_FILESYSTEM=1 -s ASYNCIFY=1 -lnodefs.js -o $@
 endif
 endif
 endif
diff --git a/src/backend/main/main.c b/src/backend/main/main.c
@@ -65,37 +65,6 @@ main(int argc, char *argv[])
 
 	printf("Hey-hey!\n");
 
-#ifdef EMSCRIPTEN
-	EM_ASM(
-		// FS.mkdir('/data');
-
-		// FS.mkdir('/usr');
-		// FS.mkdir('/usr/local');
-		// FS.mkdir('/usr/local/pgsql');
-
-		// FS.mkdir('/Users');
-		// FS.mkdir('/Users/stas');
-
-		FS.mkdir('/Users/stas/datadir/pg_notify');
-
-		// FS.mkdir('/Users/stas/datadir/pg_snapshots');
-		FS.mkdir('/Users/stas/datadir/pg_commit_ts');
-		// FS.mkdir('/Users/stas/datadir/pg_stat');
-		
-		// FS.mkdir('/Users/stas/datadir/pg_serial');
-		FS.mkdir('/Users/stas/datadir/pg_replslot');
-		// FS.mkdir('/Users/stas/datadir/pg_dynshmem');
-		FS.mkdir('/Users/stas/datadir/pg_twophase');
-		FS.mkdir('/Users/stas/datadir/pg_tblspc');
-		// FS.mkdir('/Users/stas/datadir/pg_dynshmem');
-
-		// FS.mkdir('/usr/local/pgsql/share');
-
-		// FS.mount(MEMFS, { root: '/Users/stas/datadir' }, '/data');
-		// FS.mount(MEMFS, { root: '/usr/local/pgsql/share' }, '/usr/local/pgsql/share');
-	);
-#endif
-
 	/*
 	 * If supported on the current platform, set up a handler to be called if
 	 * the backend/postmaster crashes with a fatal signal or exception.
diff --git a/src/backend/utils/cache/relcache.c b/src/backend/utils/cache/relcache.c
@@ -6485,6 +6485,16 @@ RelationCacheInitFileRemove(void)
 	struct dirent *de;
 	char		path[MAXPGPATH + 10 + sizeof(TABLESPACE_VERSION_DIRECTORY)];
 
+#ifdef EMSCRIPTEN
+	/*
+	 * emscripten 3.0.1 breaks here on traversal up in FS.realPath() on the
+	 * mount point (it has path field set, but not the name field).
+	 *
+	 * TODO: make a standalone repro and report.
+	 */
+	return;
+#endif
+
 	snprintf(path, sizeof(path), "global/%s",
 			 RELCACHE_INIT_FILENAME);
 	unlink_initfile(path, LOG);
diff --git a/wasm/.gitignore b/wasm/.gitignore
@@ -0,0 +1,2 @@
+debug
+release
diff --git a/wasm/Makefile b/wasm/Makefile
@@ -0,0 +1,26 @@
+.PHONY: debug-build debug-datadir
+
+debug-build:
+	EMCC_CFLAGS="-Wl,--allow-undefined" \
+	emconfigure ./configure CFLAGS='-Oz' \
+		--without-readline \
+		--without-zlib \
+		--disable-thread-safety \
+		--disable-spinlocks \
+		--with-system-tzdata=/usr/share/zoneinfo \
+		--enable-debug
+	EMCC_CFLAGS="-s ERROR_ON_UNDEFINED_SYMBOLS=0 -s WARN_ON_UNDEFINED_SYMBOLS=0 -s TOTAL_MEMORY=65536000 -s EMULATE_FUNCTION_POINTER_CASTS=1 -s ASSERTIONS=1" \
+		emmake make MAKELEVEL=0 -j4
+	mkdir -p wasm/debug
+	cp src/backend/postgres wasm/debug/postgres.js
+	cp src/backend/postgres.wasm wasm/debug/postgres.wasm
+
+debug-datadir:
+	mkdir -p tmp_install
+	DESTDIR="$(abspath tmp_install)" \
+	EMCC_CFLAGS="-s ERROR_ON_UNDEFINED_SYMBOLS=0 -s WARN_ON_UNDEFINED_SYMBOLS=0 -s TOTAL_MEMORY=65536000 -s EMULATE_FUNCTION_POINTER_CASTS=1 -s ASSERTIONS=1" \
+		emmake make MAKELEVEL=0 -C src/backend/ install
+	node wasm/initdb.js
+	cd wasm/debug && \
+	`em-config EMSCRIPTEN_ROOT`/tools/file_packager pgdata.data --preload ./temp_pgdata@/pgdata --js-output=pgdata.js && \
+	`em-config EMSCRIPTEN_ROOT`/tools/file_packager share.data --preload ../../tmp_install/usr/local/pgsql/share@/usr/local/pgsql/share --js-output=share.js
diff --git a/wasm/index.html b/wasm/index.html
@@ -2,7 +2,7 @@
 <html lang=en>
 <head>
 <meta charset=utf-8>
-<title>zenith</title>
+<title>pg-wasm</title>
 <link rel="preconnect" href="https://fonts.gstatic.com">
 <link href="https://fonts.googleapis.com/css2?family=Fira+Code&display=swap" rel="stylesheet">
 <link href="styles.css" rel="stylesheet">
@@ -48,9 +48,15 @@
         // };
 
         var Module = {
-          preRun: [],
+          preRun: [()=> {
+            FS.mkdir('/pgdata/pg_notify');
+            FS.mkdir('/pgdata/pg_commit_ts');
+            FS.mkdir('/pgdata/pg_replslot');
+            FS.mkdir('/pgdata/pg_twophase');
+            FS.mkdir('/pgdata/pg_tblspc');
+          }],
           postRun: [],
-          arguments: ['--single','-F','-O','-j','-c','search_path=pg_catalog','-c','dynamic_shared_memory_type=mmap','-d','0','-D','/Users/stas/datadir','template1'],
+          arguments: ['--single','-F','-O','-j','-c','search_path=pg_catalog','-c','dynamic_shared_memory_type=mmap','-d','0','-D','/pgdata','template1'],
           print: (function() {
             return function(text) {
                 if (arguments.length > 1) text = Array.prototype.slice.call(arguments).join(' ');
@@ -69,6 +75,9 @@
           monitorRunDependencies: function(left) {
             this.totalDependencies = Math.max(this.totalDependencies, left);
             Module.setStatus(left ? 'Preparing... (' + (this.totalDependencies-left) + '/' + this.totalDependencies + ')' : 'All downloads complete.');
+          },
+          locateFile: (base, _path) => {
+            return "debug/" + base;
           }
         };
 
@@ -81,9 +90,9 @@
           };
         };
       </script>
-      <script async type="text/javascript" src="share.js"></script>
-      <script async type="text/javascript" src="datadir.js"></script>
-      <script async type="text/javascript" src="postgres.js"></script>
+      <script async type="text/javascript" src="debug/share.js"></script>
+      <script async type="text/javascript" src="debug/pgdata.js"></script>
+      <script async type="text/javascript" src="debug/postgres.js"></script>
 
 </body>
 <script src="script.js"></script>
diff --git a/wasm/initdb.js b/wasm/initdb.js
@@ -0,0 +1,116 @@
+#!/usr/bin/env node
+// 
+// This script generates file packs needed for postgres to start:
+//
+// * Properly initialized data directory. We use NODEFS driver to share the
+//   data directory between the host system and wasm. That way initialization
+//   mostly happens inside of the wasm. Some preliminary steps like creating
+//   directory strycture and filling BKI file template are done in that script.
+//
+// * /usr/local/pgsql/share/ directory. Postgres needs it at least for the
+//   timezone. Probably with a bit of work we could get rid of it.
+//
+var fs = require('fs');
+var path = require('path');
+
+DIRS = [
+  'global',
+  'pg_wal',
+  'pg_wal/archive_status',
+  'pg_commit_ts',
+  'pg_dynshmem',
+  'pg_notify',
+  'pg_serial',
+  'pg_snapshots',
+  'pg_subtrans',
+  'pg_twophase',
+  'pg_multixact',
+  'pg_multixact/members',
+  'pg_multixact/offsets',
+  'base',
+  'base/1',
+  'pg_replslot',
+  'pg_tblspc',
+  'pg_stat',
+  'pg_stat_tmp',
+  'pg_xact',
+  'pg_logical',
+  'pg_logical/snapshots',
+  'pg_logical/mapping',
+]
+
+
+FILES = [
+  'postgresql.conf',
+  'postgresql.auto.conf',
+  'pg_ident.conf',
+  'pg_hba.conf',
+]
+
+WASM_PGDATA = '/pgdata'
+
+// Wasm has it's own 32-bit architecture, so wee need to initialize the data directory
+// from the wasm itself. Use NODEFS shared directories to do this.
+function initDataDir(build_path, share_path) {
+
+  let datadir_path = build_path + '/temp_pgdata'
+
+  // 1. Create postgres datadir structure
+  console.log('Creating postgres datadir structure in :', datadir_path)
+  fs.rmSync(datadir_path, {recursive: true, force: true});
+  fs.mkdirSync(datadir_path);
+  fs.chmodSync(datadir_path, '0750');
+  for (dir of DIRS) {
+    console.log('Creating directory: ' + datadir_path + '/' + dir);
+    fs.mkdirSync(datadir_path + '/' + dir);
+    fs.chmodSync(datadir_path + '/' + dir, '0700');
+  }
+  for (file of FILES) {
+    fs.writeFileSync(datadir_path + '/' + file, '');
+  }
+  fs.writeFileSync(datadir_path + '/PG_VERSION', '15devel');
+  fs.writeFileSync(datadir_path + '/base/1/PG_VERSION', '15devel');
+
+  // 2. Fill BKI file template
+  let bki = fs.readFileSync(share_path + '/postgres.bki', 'utf8')
+
+  bki = bki.replaceAll('NAMEDATALEN', '64')
+  bki = bki.replaceAll('SIZEOF_POINTER', '4')
+  bki = bki.replaceAll('ALIGNOF_POINTER', 'i')
+  bki = bki.replaceAll('FLOAT8PASSBYVAL', 'false')
+  bki = bki.replaceAll('POSTGRES', "'postgres'")
+  bki = bki.replaceAll('ENCODING', '6') // PG_UTF8
+  bki = bki.replaceAll('LC_COLLATE', "'en_US.UTF-8'")
+  bki = bki.replaceAll('LC_CTYPE', "'en_US.UTF-8'")
+
+  fs.writeFileSync(share_path + '/postgres_wasm.bki', bki);
+
+  // 3. Bootstrap postgres
+  var Module = {
+    preRun: () => {
+        FS.mkdir(WASM_PGDATA);
+        FS.mount(NODEFS, { root: datadir_path }, WASM_PGDATA);
+  
+        FS.mkdir('/usr');
+        FS.mkdir('/usr/local');
+        FS.mkdir('/usr/local/pgsql');
+        FS.mkdir('/usr/local/pgsql/share');
+        FS.mount(NODEFS, { root: share_path }, '/usr/local/pgsql/share');
+    },
+    locateFile: (file_path, _dir) => {
+      let p = path.resolve(build_path, file_path);
+      console.log('Locate file:', file_path, '->', p);
+      return p;
+    },
+    arguments: ['--boot', '-x1', '-X', '16777216', '-d', '5', '-c', 'dynamic_shared_memory_type=mmap', '-D', WASM_PGDATA]
+  };
+  eval(fs.readFileSync(path.resolve(build_path,'postgres.js')).toString());
+
+}
+
+// assumming that this script is located in repo_root/wasm
+let build_path = path.resolve(__dirname, 'debug');
+let share_path = path.resolve(__dirname, '../tmp_install/usr/local/pgsql/share');
+
+initDataDir(build_path, share_path);
+
diff --git a/wasm/script.js b/wasm/script.js
diff --git a/wasm/styles.css b/wasm/styles.css
