Skip to content

Commit d95f712

Browse files
klauspostklauspost
authored
Add github actions (#53)
* Add github actions * Skip tests on non-avx2 * Reset error mask * Remove amd64 tags for parse_number. Remove unused asm. * Update download URL * Print correct error on download failure. * Use smaller payload for NDJSON tests.
1 parent e885dec commit d95f712

24 files changed

+215
-1170
lines changed

‎.github/workflows/go.yml

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
name: Go
2+
3+
on:
4+
push:
5+
branches: [ master ]
6+
pull_request:
7+
branches: [ master ]
8+
9+
10+
jobs:
11+
build:
12+
strategy:
13+
matrix:
14+
go-version: [1.15.x, 1.16.x, 1.17.x]
15+
os: [ubuntu-latest, macos-latest, windows-latest]
16+
env:
17+
CGO_ENABLED: 0
18+
runs-on: ${{ matrix.os }}
19+
steps:
20+
- name: Set up Go
21+
uses: actions/setup-go@v2
22+
with:
23+
go-version: ${{ matrix.go-version }}
24+
25+
- name: Checkout code
26+
uses: actions/checkout@v2
27+
28+
- name: Test
29+
run: go test ./...
30+
31+
- name: Test Noasm
32+
run: go test -tags=noasm ./...
33+
34+
- name: Test Race
35+
env:
36+
CGO_ENABLED: 1
37+
run: go test -short -race ./...
38+
39+
build-special:
40+
env:
41+
CGO_ENABLED: 0
42+
runs-on: ubuntu-latest
43+
steps:
44+
- name: Set up Go
45+
uses: actions/setup-go@v2
46+
with:
47+
go-version: 1.17.x
48+
49+
- name: Checkout code
50+
uses: actions/checkout@v2
51+
52+
- name: fmt
53+
run: diff <(gofmt -d .) <(printf "")
54+
55+
- name: Vet
56+
run: go vet ./...
57+
58+
- name: Test 386
59+
run: GOOS=linux GOARCH=386 go test -short ./...

‎README.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -269,7 +269,7 @@ To serialize a block of parsed data use the [`Serialize`](https://pkg.go.dev/git
269269
To read back use the [`Deserialize`](https://pkg.go.dev/github.com/minio/simdjson-go#Serializer.Deserialize) method.
270270
For deserializing the compression mode does not need to match since it is read from the stream.
271271

272-
Example of speed for serializer/deserializer on [`parking-citations-1M`](https://files.klauspost.com/compress/parking-citations-1M.json.zst).
272+
Example of speed for serializer/deserializer on [`parking-citations-1M`](https://dl.minio.io/assets/parking-citations-1M.json.zst).
273273

274274
| Compress Mode | % of JSON size | Serialize Speed | Deserialize Speed |
275275
|---------------|----------------|-----------------|-------------------|

‎find_quote_mask_and_bits_amd64.s

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -89,16 +89,16 @@ TEXT ·_find_quote_mask_and_bits_avx512(SB), $0-48
8989
MOVQ odd_ends+8(FP), DX
9090
MOVQ prev_iter_inside_quote+16(FP), CX
9191

92-
KORQ K_ERRORMASK, K_ERRORMASK, K_ERRORMASK
92+
KXORQ K_ERRORMASK, K_ERRORMASK, K_ERRORMASK
9393

9494
VMOVDQU32 (DI), Z8
9595

9696
CALL ·__init_quote_mask_and_bits_avx512(SB)
9797
CALL ·__find_quote_mask_and_bits_avx512(SB)
9898

99-
VZEROUPPER
10099
KMOVQ K_ERRORMASK, error_mask+24(FP)
101100
KMOVQ K_QUOTEBITS, quote_bits+32(FP)
101+
VZEROUPPER
102102
MOVQ AX, quote_mask+40(FP)
103103
RET
104104

‎find_subroutines_amd64.go

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
1-
//+build !noasm
2-
//+build !appengine
3-
//+build gc
1+
//go:build !noasm && !appengine && gc
2+
// +build !noasm,!appengine,gc
43

54
/*
65
* MinIO Cloud Storage, (C) 2020 MinIO, Inc.

‎find_subroutines_amd64_test.go

Lines changed: 49 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
1-
//+build !noasm
2-
//+build !appengine
3-
//+build gc
1+
//go:build !noasm && !appengine && gc
2+
// +build !noasm,!appengine,gc
43

54
/*
65
* MinIO Cloud Storage, (C) 2020 MinIO, Inc.
@@ -31,7 +30,9 @@ import (
3130
)
3231

3332
func TestFinalizeStructurals(t *testing.T) {
34-
33+
if !SupportedCPU() {
34+
t.SkipNow()
35+
}
3536
testCases := []struct {
3637
structurals uint64
3738
whitespace uint64
@@ -92,6 +93,9 @@ func testFindNewlineDelimiters(t *testing.T, f func([]byte, uint64) uint64) {
9293
}
9394

9495
func TestFindNewlineDelimiters(t *testing.T) {
96+
if !SupportedCPU() {
97+
t.SkipNow()
98+
}
9599
t.Run("avx2", func(t *testing.T) {
96100
testFindNewlineDelimiters(t, _find_newline_delimiters)
97101
})
@@ -103,7 +107,9 @@ func TestFindNewlineDelimiters(t *testing.T) {
103107
}
104108

105109
func testExcludeNewlineDelimitersWithinQuotes(t *testing.T, f func([]byte, uint64) uint64) {
106-
110+
if !SupportedCPU() {
111+
t.SkipNow()
112+
}
107113
input := []byte(` "-------------------------------------" `)
108114
input[10] = 0x0a // within quoted string, so should be ignored
109115
input[50] = 0x0a // outside quoted string, so should be found
@@ -122,6 +128,9 @@ func testExcludeNewlineDelimitersWithinQuotes(t *testing.T, f func([]byte, uint6
122128
}
123129

124130
func TestExcludeNewlineDelimitersWithinQuotes(t *testing.T) {
131+
if !SupportedCPU() {
132+
t.SkipNow()
133+
}
125134
t.Run("avx2", func(t *testing.T) {
126135
testExcludeNewlineDelimitersWithinQuotes(t, _find_newline_delimiters)
127136
})
@@ -190,6 +199,9 @@ func testFindOddBackslashSequences(t *testing.T, f func([]byte, *uint64) uint64)
190199
}
191200

192201
func TestFindOddBackslashSequences(t *testing.T) {
202+
if !SupportedCPU() {
203+
t.SkipNow()
204+
}
193205
t.Run("avx2", func(t *testing.T) {
194206
testFindOddBackslashSequences(t, find_odd_backslash_sequences)
195207
})
@@ -291,6 +303,9 @@ func testFindQuoteMaskAndBits(t *testing.T, f func([]byte, uint64, *uint64, *uin
291303
}
292304

293305
func TestFindQuoteMaskAndBits(t *testing.T) {
306+
if !SupportedCPU() {
307+
t.SkipNow()
308+
}
294309
t.Run("avx2", func(t *testing.T) {
295310
testFindQuoteMaskAndBits(t, find_quote_mask_and_bits)
296311
})
@@ -346,6 +361,9 @@ func testFindStructuralBits(t *testing.T, f func([]byte, *uint64, *uint64, *uint
346361
}
347362

348363
func TestFindStructuralBits(t *testing.T) {
364+
if !SupportedCPU() {
365+
t.SkipNow()
366+
}
349367
t.Run("avx2", func(t *testing.T) {
350368
testFindStructuralBits(t, find_structural_bits)
351369
})
@@ -404,6 +422,9 @@ func testFindStructuralBitsWhitespacePadding(t *testing.T, f func([]byte, *uint6
404422
}
405423

406424
func TestFindStructuralBitsWhitespacePadding(t *testing.T) {
425+
if !SupportedCPU() {
426+
t.SkipNow()
427+
}
407428
t.Run("avx2", func(t *testing.T) {
408429
testFindStructuralBitsWhitespacePadding(t, find_structural_bits_in_slice)
409430
})
@@ -459,6 +480,9 @@ func testFindStructuralBitsLoop(t *testing.T, f func([]byte, *uint64, *uint64, *
459480
}
460481

461482
func TestFindStructuralBitsLoop(t *testing.T) {
483+
if !SupportedCPU() {
484+
t.SkipNow()
485+
}
462486
t.Run("avx2", func(t *testing.T) {
463487
testFindStructuralBitsLoop(t, find_structural_bits_in_slice)
464488
})
@@ -491,6 +515,9 @@ func benchmarkFindStructuralBits(b *testing.B, f func([]byte, *uint64, *uint64,
491515
}
492516

493517
func BenchmarkFindStructuralBits(b *testing.B) {
518+
if !SupportedCPU() {
519+
b.SkipNow()
520+
}
494521
b.Run("avx2", func(b *testing.B) {
495522
benchmarkFindStructuralBits(b, find_structural_bits)
496523
})
@@ -531,6 +558,9 @@ func benchmarkFindStructuralBitsLoop(b *testing.B, f func([]byte, *uint64, *uint
531558
}
532559

533560
func BenchmarkFindStructuralBitsLoop(b *testing.B) {
561+
if !SupportedCPU() {
562+
b.SkipNow()
563+
}
534564
b.Run("avx2", func(b *testing.B) {
535565
benchmarkFindStructuralBitsLoop(b, find_structural_bits_in_slice)
536566
})
@@ -578,6 +608,10 @@ func benchmarkFindStructuralBitsParallelLoop(b *testing.B, f func([]byte, *uint6
578608
}
579609

580610
func BenchmarkFindStructuralBitsParallelLoop(b *testing.B) {
611+
if !SupportedCPU() {
612+
b.SkipNow()
613+
}
614+
581615
b.Run("avx2", func(b *testing.B) {
582616
benchmarkFindStructuralBitsParallelLoop(b, find_structural_bits_in_slice)
583617
})
@@ -655,6 +689,10 @@ func testFindWhitespaceAndStructurals(t *testing.T, f func([]byte, *uint64, *uin
655689
}
656690

657691
func TestFindWhitespaceAndStructurals(t *testing.T) {
692+
if !SupportedCPU() {
693+
t.SkipNow()
694+
}
695+
658696
t.Run("avx2", func(t *testing.T) {
659697
testFindWhitespaceAndStructurals(t, find_whitespace_and_structurals)
660698
})
@@ -666,6 +704,9 @@ func TestFindWhitespaceAndStructurals(t *testing.T) {
666704
}
667705

668706
func TestFlattenBitsIncremental(t *testing.T) {
707+
if !SupportedCPU() {
708+
t.SkipNow()
709+
}
669710

670711
testCases := []struct {
671712
masks []uint64
@@ -732,6 +773,9 @@ func TestFlattenBitsIncremental(t *testing.T) {
732773
}
733774

734775
func BenchmarkFlattenBits(b *testing.B) {
776+
if !SupportedCPU() {
777+
b.SkipNow()
778+
}
735779

736780
msg := loadCompressed(b, "twitter")
737781

‎ndjson_test.go

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -254,13 +254,16 @@ func TestNdjsonCountWhere(t *testing.T) {
254254
if testing.Short() {
255255
t.Skip("skipping... too long")
256256
}
257-
ndjson := loadFile("testdata/parking-citations-1M.json.zst")
257+
ndjson := loadFile("testdata/parking-citations.json.zst")
258258
pj, err := ParseND(ndjson, nil)
259259
if err != nil {
260260
t.Fatal(err)
261261
}
262-
263-
const want = 110349
262+
ser := NewSerializer()
263+
ser.CompressMode(CompressBest)
264+
b := ser.Serialize(nil, *pj)
265+
t.Log(len(b))
266+
const want = 116
264267
if result := countWhere("Make", "HOND", *pj); result != want {
265268
t.Errorf("TestNdjsonCountWhere: got: %d want: %d", result, want)
266269
}
@@ -306,19 +309,21 @@ func loadFile(filename string) []byte {
306309
break
307310
}
308311
if os.IsNotExist(err) {
309-
fmt.Println("downloading file" + filename)
310-
resp, err := http.DefaultClient.Get("https://files.klauspost.com/compress/" + filepath.Base(filename))
312+
fmt.Println("downloading file", filename)
313+
resp, err := http.DefaultClient.Get("https://dl.minio.io/assets/" + filepath.Base(filename))
311314
if err == nil && resp.StatusCode == http.StatusOK {
312315
b, err := ioutil.ReadAll(resp.Body)
313316
if err == nil {
314317
err = ioutil.WriteFile(filename, b, os.ModePerm)
315318
if err == nil {
316319
continue
317320
}
321+
panic("Failed to write file:" + err.Error())
318322
}
323+
panic("Failed to read file:" + err.Error())
319324
}
325+
panic("Failed to download file:" + err.Error())
320326
}
321-
panic("Failed to (down)load file:" + err.Error())
322327
}
323328
dec, err := zstd.NewReader(f)
324329
if err != nil {

‎parse_json_amd64.go

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
1-
//+build !noasm
2-
//+build !appengine
3-
//+build gc
1+
//go:build !noasm && !appengine && gc
2+
// +build !noasm,!appengine,gc
43

54
/*
65
* MinIO Cloud Storage, (C) 2020 MinIO, Inc.

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy