Bump github.com/blevesearch/bleve/v2 from 2.3.7 to 2.3.9

Bumps [github.com/blevesearch/bleve/v2](https://github.com/blevesearch/bleve) from 2.3.7 to 2.3.9. - [Release notes](https://github.com/blevesearch/bleve/releases) - [Commits](https://github.com/blevesearch/bleve/compare/v2.3.7...v2.3.9) --- updated-dependencies: - dependency-name: github.com/blevesearch/bleve/v2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] <support@github.com>
2023-08-15 11:00:48 +02:00
parent 82b600aef5
commit f9b69afa9e
58 changed files with 1138 additions and 330 deletions
@@ -1,3 +1,4 @@
+//go:build (386 && !appengine) || (amd64 && !appengine) || (arm && !appengine) || (arm64 && !appengine) || (ppc64le && !appengine) || (mipsle && !appengine) || (mips64le && !appengine) || (mips64p32le && !appengine) || (wasm && !appengine)
 // +build 386,!appengine amd64,!appengine arm,!appengine arm64,!appengine ppc64le,!appengine mipsle,!appengine mips64le,!appengine mips64p32le,!appengine wasm,!appengine

 package roaring
@@ -56,6 +57,22 @@ func uint16SliceAsByteSlice(slice []uint16) []byte {
 	return result
 }

+func interval16SliceAsByteSlice(slice []interval16) []byte {
+	// make a new slice header
+	header := *(*reflect.SliceHeader)(unsafe.Pointer(&slice))
+
+	// update its capacity and length
+	header.Len *= 4
+	header.Cap *= 4
+
+	// instantiate result and use KeepAlive so data isn't unmapped.
+	result := *(*[]byte)(unsafe.Pointer(&header))
+	runtime.KeepAlive(&slice)
+
+	// return it
+	return result
+}
+
 func (bc *bitmapContainer) asLittleEndianByteSlice() []byte {
 	return uint64SliceAsByteSlice(bc.bitmap)
 }
@@ -134,7 +151,124 @@ func byteSliceAsInterval16Slice(slice []byte) (result []interval16) {
 	return
 }

-// FromBuffer creates a bitmap from its serialized version stored in buffer.
+func byteSliceAsContainerSlice(slice []byte) (result []container) {
+	var c container
+	containerSize := int(unsafe.Sizeof(c))
+
+	if len(slice)%containerSize != 0 {
+		panic("Slice size should be divisible by unsafe.Sizeof(container)")
+	}
+	// reference: https://go101.org/article/unsafe.html
+
+	// make a new slice header
+	bHeader := (*reflect.SliceHeader)(unsafe.Pointer(&slice))
+	rHeader := (*reflect.SliceHeader)(unsafe.Pointer(&result))
+
+	// transfer the data from the given slice to a new variable (our result)
+	rHeader.Data = bHeader.Data
+	rHeader.Len = bHeader.Len / containerSize
+	rHeader.Cap = bHeader.Cap / containerSize
+
+	// instantiate result and use KeepAlive so data isn't unmapped.
+	runtime.KeepAlive(&slice) // it is still crucial, GC can free it)
+
+	// return result
+	return
+}
+
+func byteSliceAsBitsetSlice(slice []byte) (result []bitmapContainer) {
+	bitsetSize := int(unsafe.Sizeof(bitmapContainer{}))
+	if len(slice)%bitsetSize != 0 {
+		panic("Slice size should be divisible by unsafe.Sizeof(bitmapContainer)")
+	}
+	// reference: https://go101.org/article/unsafe.html
+
+	// make a new slice header
+	bHeader := (*reflect.SliceHeader)(unsafe.Pointer(&slice))
+	rHeader := (*reflect.SliceHeader)(unsafe.Pointer(&result))
+
+	// transfer the data from the given slice to a new variable (our result)
+	rHeader.Data = bHeader.Data
+	rHeader.Len = bHeader.Len / bitsetSize
+	rHeader.Cap = bHeader.Cap / bitsetSize
+
+	// instantiate result and use KeepAlive so data isn't unmapped.
+	runtime.KeepAlive(&slice) // it is still crucial, GC can free it)
+
+	// return result
+	return
+}
+
+func byteSliceAsArraySlice(slice []byte) (result []arrayContainer) {
+	arraySize := int(unsafe.Sizeof(arrayContainer{}))
+	if len(slice)%arraySize != 0 {
+		panic("Slice size should be divisible by unsafe.Sizeof(arrayContainer)")
+	}
+	// reference: https://go101.org/article/unsafe.html
+
+	// make a new slice header
+	bHeader := (*reflect.SliceHeader)(unsafe.Pointer(&slice))
+	rHeader := (*reflect.SliceHeader)(unsafe.Pointer(&result))
+
+	// transfer the data from the given slice to a new variable (our result)
+	rHeader.Data = bHeader.Data
+	rHeader.Len = bHeader.Len / arraySize
+	rHeader.Cap = bHeader.Cap / arraySize
+
+	// instantiate result and use KeepAlive so data isn't unmapped.
+	runtime.KeepAlive(&slice) // it is still crucial, GC can free it)
+
+	// return result
+	return
+}
+
+func byteSliceAsRun16Slice(slice []byte) (result []runContainer16) {
+	run16Size := int(unsafe.Sizeof(runContainer16{}))
+	if len(slice)%run16Size != 0 {
+		panic("Slice size should be divisible by unsafe.Sizeof(runContainer16)")
+	}
+	// reference: https://go101.org/article/unsafe.html
+
+	// make a new slice header
+	bHeader := (*reflect.SliceHeader)(unsafe.Pointer(&slice))
+	rHeader := (*reflect.SliceHeader)(unsafe.Pointer(&result))
+
+	// transfer the data from the given slice to a new variable (our result)
+	rHeader.Data = bHeader.Data
+	rHeader.Len = bHeader.Len / run16Size
+	rHeader.Cap = bHeader.Cap / run16Size
+
+	// instantiate result and use KeepAlive so data isn't unmapped.
+	runtime.KeepAlive(&slice) // it is still crucial, GC can free it)
+
+	// return result
+	return
+}
+
+func byteSliceAsBoolSlice(slice []byte) (result []bool) {
+	boolSize := int(unsafe.Sizeof(true))
+	if len(slice)%boolSize != 0 {
+		panic("Slice size should be divisible by unsafe.Sizeof(bool)")
+	}
+	// reference: https://go101.org/article/unsafe.html
+
+	// make a new slice header
+	bHeader := (*reflect.SliceHeader)(unsafe.Pointer(&slice))
+	rHeader := (*reflect.SliceHeader)(unsafe.Pointer(&result))
+
+	// transfer the data from the given slice to a new variable (our result)
+	rHeader.Data = bHeader.Data
+	rHeader.Len = bHeader.Len / boolSize
+	rHeader.Cap = bHeader.Cap / boolSize
+
+	// instantiate result and use KeepAlive so data isn't unmapped.
+	runtime.KeepAlive(&slice) // it is still crucial, GC can free it)
+
+	// return result
+	return
+}
+
+// FrozenView creates a static view of a serialized bitmap stored in buf.
 // It uses CRoaring's frozen bitmap format.
 //
 // The format specification is available here:
@@ -198,13 +332,13 @@ func (rb *Bitmap) FrozenView(buf []byte) error {
 const FROZEN_COOKIE = 13766

 var (
-	FrozenBitmapInvalidCookie = errors.New("header does not contain the FROZEN_COOKIE")
-	FrozenBitmapBigEndian = errors.New("loading big endian frozen bitmaps is not supported")
-	FrozenBitmapIncomplete = errors.New("input buffer too small to contain a frozen bitmap")
-	FrozenBitmapOverpopulated = errors.New("too many containers")
-	FrozenBitmapUnexpectedData = errors.New("spurious data in input")
+	FrozenBitmapInvalidCookie   = errors.New("header does not contain the FROZEN_COOKIE")
+	FrozenBitmapBigEndian       = errors.New("loading big endian frozen bitmaps is not supported")
+	FrozenBitmapIncomplete      = errors.New("input buffer too small to contain a frozen bitmap")
+	FrozenBitmapOverpopulated   = errors.New("too many containers")
+	FrozenBitmapUnexpectedData  = errors.New("spurious data in input")
 	FrozenBitmapInvalidTypecode = errors.New("unrecognized typecode")
-	FrozenBitmapBufferTooSmall = errors.New("buffer too small")
+	FrozenBitmapBufferTooSmall  = errors.New("buffer too small")
 )

 func (ra *roaringArray) frozenView(buf []byte) error {
@@ -213,14 +347,14 @@ func (ra *roaringArray) frozenView(buf []byte) error {
 	}

 	headerBE := binary.BigEndian.Uint32(buf[len(buf)-4:])
-	if headerBE & 0x7fff == FROZEN_COOKIE {
+	if headerBE&0x7fff == FROZEN_COOKIE {
 		return FrozenBitmapBigEndian
 	}

 	header := binary.LittleEndian.Uint32(buf[len(buf)-4:])
 	buf = buf[:len(buf)-4]

-	if header & 0x7fff != FROZEN_COOKIE {
+	if header&0x7fff != FROZEN_COOKIE {
 		return FrozenBitmapInvalidCookie
 	}

@@ -243,29 +377,29 @@ func (ra *roaringArray) frozenView(buf []byte) error {
 	keys := byteSliceAsUint16Slice(buf[len(buf)-2*nCont:])
 	buf = buf[:len(buf)-2*nCont]

-	nBitmap, nArray, nRun := uint64(0), uint64(0), uint64(0)
-	nArrayEl, nRunEl := uint64(0), uint64(0)
+	nBitmap, nArray, nRun := 0, 0, 0
+	nArrayEl, nRunEl := 0, 0
 	for i, t := range types {
-		switch (t) {
+		switch t {
 		case 1:
 			nBitmap++
 		case 2:
 			nArray++
-			nArrayEl += uint64(counts[i])+1
+			nArrayEl += int(counts[i]) + 1
 		case 3:
 			nRun++
-			nRunEl += uint64(counts[i])
+			nRunEl += int(counts[i])
 		default:
 			return FrozenBitmapInvalidTypecode
 		}
 	}

-	if uint64(len(buf)) < (1 << 13)*nBitmap + 4*nRunEl + 2*nArrayEl {
+	if len(buf) < (1<<13)*nBitmap+4*nRunEl+2*nArrayEl {
 		return FrozenBitmapIncomplete
 	}

-	bitsetsArena := byteSliceAsUint64Slice(buf[:(1 << 13)*nBitmap])
-	buf = buf[(1 << 13)*nBitmap:]
+	bitsetsArena := byteSliceAsUint64Slice(buf[:(1<<13)*nBitmap])
+	buf = buf[(1<<13)*nBitmap:]

 	runsArena := byteSliceAsInterval16Slice(buf[:4*nRunEl])
 	buf = buf[4*nRunEl:]
@@ -277,27 +411,44 @@ func (ra *roaringArray) frozenView(buf []byte) error {
 		return FrozenBitmapUnexpectedData
 	}

-	// TODO: maybe arena_alloc all this.
-	containers := make([]container, nCont)
-	bitsets := make([]bitmapContainer, nBitmap)
-	arrays := make([]arrayContainer, nArray)
-	runs := make([]runContainer16, nRun)
-	needCOW := make([]bool, nCont)
+	var c container
+	containersSz := int(unsafe.Sizeof(c))*nCont
+	bitsetsSz := int(unsafe.Sizeof(bitmapContainer{}))*nBitmap
+	arraysSz := int(unsafe.Sizeof(arrayContainer{}))*nArray
+	runsSz := int(unsafe.Sizeof(runContainer16{}))*nRun
+	needCOWSz := int(unsafe.Sizeof(true))*nCont

-	iBitset, iArray, iRun := uint64(0), uint64(0), uint64(0)
+	bitmapArenaSz := containersSz + bitsetsSz + arraysSz + runsSz + needCOWSz
+	bitmapArena := make([]byte, bitmapArenaSz)
+
+	containers := byteSliceAsContainerSlice(bitmapArena[:containersSz])
+	bitmapArena = bitmapArena[containersSz:]
+
+	bitsets := byteSliceAsBitsetSlice(bitmapArena[:bitsetsSz])
+	bitmapArena = bitmapArena[bitsetsSz:]
+
+	arrays := byteSliceAsArraySlice(bitmapArena[:arraysSz])
+	bitmapArena = bitmapArena[arraysSz:]
+
+	runs := byteSliceAsRun16Slice(bitmapArena[:runsSz])
+	bitmapArena = bitmapArena[runsSz:]
+
+	needCOW := byteSliceAsBoolSlice(bitmapArena)
+
+	iBitset, iArray, iRun := 0, 0, 0
 	for i, t := range types {
 		needCOW[i] = true

-		switch (t) {
+		switch t {
 		case 1:
 			containers[i] = &bitsets[iBitset]
-			bitsets[iBitset].cardinality = int(counts[i])+1
+			bitsets[iBitset].cardinality = int(counts[i]) + 1
 			bitsets[iBitset].bitmap = bitsetsArena[:1024]
 			bitsetsArena = bitsetsArena[1024:]
 			iBitset++
 		case 2:
 			containers[i] = &arrays[iArray]
-			sz := int(counts[i])+1
+			sz := int(counts[i]) + 1
 			arrays[iArray].content = arraysArena[:sz]
 			arraysArena = arraysArena[sz:]
 			iArray++
@@ -363,13 +514,13 @@ func (bm *Bitmap) FreezeTo(buf []byte) (int, error) {
 		}
 	}

-	serialSize := 4 + 5*nCont + (1 << 13)*nBits + 4*nRunEl + 2*nArrayEl
+	serialSize := 4 + 5*nCont + (1<<13)*nBits + 4*nRunEl + 2*nArrayEl
 	if len(buf) < serialSize {
 		return 0, FrozenBitmapBufferTooSmall
 	}

-	bitsArena := byteSliceAsUint64Slice(buf[:(1 << 13)*nBits])
-	buf = buf[(1 << 13)*nBits:]
+	bitsArena := byteSliceAsUint64Slice(buf[:(1<<13)*nBits])
+	buf = buf[(1<<13)*nBits:]

 	runsArena := byteSliceAsInterval16Slice(buf[:4*nRunEl])
 	buf = buf[4*nRunEl:]
@@ -386,7 +537,7 @@ func (bm *Bitmap) FreezeTo(buf []byte) (int, error) {
 	types := buf[:nCont]
 	buf = buf[nCont:]

-	header := uint32(FROZEN_COOKIE|(nCont << 15))
+	header := uint32(FROZEN_COOKIE | (nCont << 15))
 	binary.LittleEndian.PutUint32(buf[:4], header)

 	copy(keys, bm.highlowcontainer.keys[:])
@@ -396,13 +547,13 @@ func (bm *Bitmap) FreezeTo(buf []byte) (int, error) {
 		case *bitmapContainer:
 			copy(bitsArena, v.bitmap)
 			bitsArena = bitsArena[1024:]
-			counts[i] = uint16(v.cardinality-1)
+			counts[i] = uint16(v.cardinality - 1)
 			types[i] = 1
 		case *arrayContainer:
 			copy(arraysArena, v.content)
 			arraysArena = arraysArena[len(v.content):]
 			elems := len(v.content)
-			counts[i] = uint16(elems-1)
+			counts[i] = uint16(elems - 1)
 			types[i] = 2
 		case *runContainer16:
 			copy(runsArena, v.iv)
@@ -415,3 +566,87 @@ func (bm *Bitmap) FreezeTo(buf []byte) (int, error) {

 	return serialSize, nil
 }
+
+func (bm *Bitmap) WriteFrozenTo(wr io.Writer) (int, error) {
+	// FIXME: this is a naive version that iterates 4 times through the
+	// containers and allocates 3*len(containers) bytes; it's quite likely
+	// it can be done more efficiently.
+	containers := bm.highlowcontainer.containers
+	written := 0
+
+	for _, c := range containers {
+		c, ok := c.(*bitmapContainer)
+		if !ok {
+			continue
+		}
+		n, err := wr.Write(uint64SliceAsByteSlice(c.bitmap))
+		written += n
+		if err != nil {
+			return written, err
+		}
+	}
+
+	for _, c := range containers {
+		c, ok := c.(*runContainer16)
+		if !ok {
+			continue
+		}
+		n, err := wr.Write(interval16SliceAsByteSlice(c.iv))
+		written += n
+		if err != nil {
+			return written, err
+		}
+	}
+
+	for _, c := range containers {
+		c, ok := c.(*arrayContainer)
+		if !ok {
+			continue
+		}
+		n, err := wr.Write(uint16SliceAsByteSlice(c.content))
+		written += n
+		if err != nil {
+			return written, err
+		}
+	}
+
+	n, err := wr.Write(uint16SliceAsByteSlice(bm.highlowcontainer.keys))
+	written += n
+	if err != nil {
+		return written, err
+	}
+
+	countTypeBuf := make([]byte, 3*len(containers))
+	counts := byteSliceAsUint16Slice(countTypeBuf[:2*len(containers)])
+	types := countTypeBuf[2*len(containers):]
+
+	for i, c := range containers {
+		switch c := c.(type) {
+		case *bitmapContainer:
+			counts[i] = uint16(c.cardinality - 1)
+			types[i] = 1
+		case *arrayContainer:
+			elems := len(c.content)
+			counts[i] = uint16(elems - 1)
+			types[i] = 2
+		case *runContainer16:
+			runs := len(c.iv)
+			counts[i] = uint16(runs)
+			types[i] = 3
+		}
+	}
+
+	n, err = wr.Write(countTypeBuf)
+	written += n
+	if err != nil {
+		return written, err
+	}
+
+	header := uint32(FROZEN_COOKIE | (len(containers) << 15))
+	if err := binary.Write(wr, binary.LittleEndian, header); err != nil {
+		return written, err
+	}
+	written += 4
+
+	return written, nil
+}