7 Commits

Author SHA1 Message Date
brent saner
2edbc9306d v1.15.4
FIXED:
* Docs error
2026-01-07 19:15:21 -05:00
brent saner
bb71be187f v1.15.3
FIXED:
* Properly parse into map, add *All* variants
2026-01-07 19:02:52 -05:00
brent saner
834395c050 v1.15.2
ADDED:
* Better docs for remap
* Added returner convenience funcs for remap

FIXED:
* Proper resliced remap.ReMap.MapString
2026-01-06 02:54:38 -05:00
brent saner
ef56898d6b v1.15.1
ADDED:
* timex, for some floaty-UNIX-y things
2025-12-23 18:57:28 -05:00
brent saner
006cf39fa1 v1.15.0
ADDED:
* tplx, for one-shotting/shortcutting templating
2025-12-23 17:26:50 -05:00
brent saner
145c32268e v1.14.0
ADDED:
* iox package
* mapsx package
* netx/inetcksum package
2025-12-18 04:47:31 -05:00
brent saner
6ddfcdb416 v1.13.0
ADDED:
* stringsx functions
2025-11-30 16:53:56 -05:00
38 changed files with 2443 additions and 302 deletions

10
go.mod
View File

@@ -1,16 +1,16 @@
module r00t2.io/goutils module r00t2.io/goutils
go 1.24.5 go 1.25
require ( require (
github.com/coreos/go-systemd/v22 v22.5.0 github.com/coreos/go-systemd/v22 v22.6.0
github.com/google/uuid v1.6.0 github.com/google/uuid v1.6.0
go4.org/netipx v0.0.0-20231129151722-fdeea329fbba go4.org/netipx v0.0.0-20231129151722-fdeea329fbba
golang.org/x/sys v0.34.0 golang.org/x/sys v0.39.0
r00t2.io/sysutils v1.14.0 r00t2.io/sysutils v1.15.1
) )
require ( require (
github.com/djherbis/times v1.6.0 // indirect github.com/djherbis/times v1.6.0 // indirect
golang.org/x/sync v0.16.0 // indirect golang.org/x/sync v0.19.0 // indirect
) )

17
go.sum
View File

@@ -1,16 +1,15 @@
github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= github.com/coreos/go-systemd/v22 v22.6.0 h1:aGVa/v8B7hpb0TKl0MWoAavPDmHvobFe5R5zn0bCJWo=
github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/go-systemd/v22 v22.6.0/go.mod h1:iG+pp635Fo7ZmV/j14KUcmEyWF+0X7Lua8rrTWzYgWU=
github.com/djherbis/times v1.6.0 h1:w2ctJ92J8fBvWPxugmXIv7Nz7Q3iDMKNx9v5ocVH20c= github.com/djherbis/times v1.6.0 h1:w2ctJ92J8fBvWPxugmXIv7Nz7Q3iDMKNx9v5ocVH20c=
github.com/djherbis/times v1.6.0/go.mod h1:gOHeRAz2h+VJNZ5Gmc/o7iD9k4wW7NMVqieYCY99oc0= github.com/djherbis/times v1.6.0/go.mod h1:gOHeRAz2h+VJNZ5Gmc/o7iD9k4wW7NMVqieYCY99oc0=
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
go4.org/netipx v0.0.0-20231129151722-fdeea329fbba h1:0b9z3AuHCjxk0x/opv64kcgZLBseWJUpBw5I82+2U4M= go4.org/netipx v0.0.0-20231129151722-fdeea329fbba h1:0b9z3AuHCjxk0x/opv64kcgZLBseWJUpBw5I82+2U4M=
go4.org/netipx v0.0.0-20231129151722-fdeea329fbba/go.mod h1:PLyyIXexvUFg3Owu6p/WfdlivPbZJsZdgWZlrGope/Y= go4.org/netipx v0.0.0-20231129151722-fdeea329fbba/go.mod h1:PLyyIXexvUFg3Owu6p/WfdlivPbZJsZdgWZlrGope/Y=
golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk=
golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
r00t2.io/sysutils v1.14.0 h1:Lrio3uPi9CuUdg+sg3WkVV1CK/qcOpV9GdFCGFG1KJs= r00t2.io/sysutils v1.15.1 h1:0EVZZAxTFqQN6jjfjqUKkXye0LMshUA5MO7l3Wd6wH8=
r00t2.io/sysutils v1.14.0/go.mod h1:ZJ7gZxFVQ7QIokQ5fPZr7wl0XO5Iu+LqtE8j3ciRINw= r00t2.io/sysutils v1.15.1/go.mod h1:T0iOnaZaSG5NE1hbXTqojRZc0ia/u8TB73lV7zhMz58=

View File

@@ -1,4 +1,7 @@
/* /*
Package iox includes extensions to the stdlib `io` module. Package iox includes extensions to the stdlib `io` module.
Not everything in here is considered fully stabilized yet,
but it should be usable.
*/ */
package iox package iox

View File

@@ -5,5 +5,13 @@ import (
) )
var ( var (
ErrBufTooSmall error = errors.New("buffer too small; buffer size must be > 0") ErrBufTooSmall error = errors.New("buffer too small; buffer size must be > 0")
ErrChunkTooBig error = errors.New("chunk too big for method")
ErrChunkTooSmall error = errors.New("chunk too small for buffer")
ErrInvalidChunkSize error = errors.New("an invalid chunk size was passed")
ErrNilCtx error = errors.New("a nil context was passed")
ErrNilReader error = errors.New("a nil reader was passed")
ErrNilWriter error = errors.New("a nil writer was passed")
ErrShortRead error = errors.New("a read was cut short with no EOF")
ErrShortWrite error = errors.New("a write was cut short with no error")
) )

View File

@@ -1,20 +1,21 @@
package iox package iox
import ( import (
`context`
`io` `io`
) )
/* /*
CopyBufN is a mix between io.CopyN and io.CopyBuffer. CopyBufN is a mix between [io.CopyN] and [io.CopyBuffer].
Despite what the docs may suggest, io.CopyN does NOT *read* n bytes from src AND write n bytes to dst. Despite what the docs may suggest, [io.CopyN] does NOT *read* n bytes from src AND write n bytes to dst.
Instead, it always reads 32 KiB from src, and writes n bytes to dst. Instead, it always reads 32 KiB from src, and writes n bytes to dst.
There are, of course, cases where this is deadfully undesired. There are cases where this is dreadfully undesired.
One can, of course, use io.CopyBuffer, but this is a bit annoying since you then have to provide a buffer yourself. One can, of course, use [io.CopyBuffer], but this is a bit annoying since you then have to provide a buffer yourself.
This convenience-wraps io.CopyBuffer to have a similar signature to io.CopyN but properly uses n for both reading and writing. This convenience-wraps [io.CopyBuffer] to have a similar signature to [io.CopyN] but properly uses n for both reading and writing.
*/ */
func CopyBufN(dst io.Writer, src io.Reader, n int64) (written int64, err error) { func CopyBufN(dst io.Writer, src io.Reader, n int64) (written int64, err error) {
@@ -32,10 +33,215 @@ func CopyBufN(dst io.Writer, src io.Reader, n int64) (written int64, err error)
return return
} }
// CopyBufWith allows for specifying a buffer allocator function, otherwise acts as CopyBufN. // CopyCtxBufN copies from `src` to `dst`, `n` bytes at a time, interruptible by `ctx`.
func CopyBufWith(dst io.Writer, src io.Reader, bufFunc func() (b []byte)) (written int64, err error) { func CopyCtxBufN(ctx context.Context, dst io.Writer, src io.Reader, n int64) (written int64, err error) {
written, err = io.CopyBuffer(dst, src, bufFunc()) var nr int
var nw int
var end bool
var buf []byte
if ctx == nil {
err = ErrNilCtx
return
}
if n <= 0 {
err = ErrBufTooSmall
return
}
endCopy:
for {
select {
case <-ctx.Done():
err = ctx.Err()
return
default:
buf = make([]byte, n)
nr, err = src.Read(buf)
if err == io.EOF {
err = nil
end = true
} else if err != nil {
return
}
buf = buf[:nr]
if nw, err = dst.Write(buf); err != nil {
written += int64(nw)
return
}
written += int64(nw)
if len(buf) != nw {
err = io.ErrShortWrite
return
}
if end {
break endCopy
}
}
}
return return
} }
/*
CopyBufWith allows for specifying a buffer allocator function, otherwise acts as [CopyBufN].
bufFunc *MUST NOT* return a nil or len == 0 buffer. [ErrBufTooSmall] will be returned if it does.
This uses a fixed buffer size from a single call to `bufFunc`.
If you need something with dynamic buffer sizing according to some state, use [CopyBufWithDynamic] instead.
(Note that CopyBufWithDynamic is generally a little slower, but it should only be noticeable on very large amounts of data.)
*/
func CopyBufWith(dst io.Writer, src io.Reader, bufFunc func() (b []byte)) (written int64, err error) {
var buf []byte = bufFunc()
if buf == nil || len(buf) == 0 {
err = ErrBufTooSmall
return
}
written, err = io.CopyBuffer(dst, src, buf)
return
}
/*
CopyBufWithDynamic is like [CopyBufWith] except it will call bufFunc after each previous buffer is written.
That is to say (using a particularly contrived example):
import time
func dynBuf() (b []byte) {
var t time.Time = time.Now()
b = make([]byte, t.Seconds())
return
}
Then:
CopyBufWithDynamic(w, r, dynBuf)
will use a buffer sized to the seconds of the time it reads in/writes out the next buffer, whereas with [CopyBufWith]:
CopyBufWith(w, r, dynBuf)
would use a *fixed* buffer size of whatever the seconds was equal to at the time of the *first call* to dynBuf.
`src` MUST return an [io.EOF] when its end is reached, but (as per e.g. [io.CopyBuffer]) the io.EOF error will not
be returned from CopyBufWithDynamic. (Any/all other errors encountered will be returned, however, and copying will
immediately cease.)
*/
func CopyBufWithDynamic(dst io.Writer, src io.Reader, bufFunc func() (b []byte)) (written int64, err error) {
var nr int
var nw int
var end bool
var buf []byte
for {
buf = bufFunc()
if buf == nil || len(buf) == 0 {
err = ErrBufTooSmall
return
}
nr, err = src.Read(buf)
if err == io.EOF {
err = nil
end = true
} else if err != nil {
return
}
buf = buf[:nr]
if nw, err = dst.Write(buf); err != nil {
written += int64(nw)
return
}
written += int64(nw)
if len(buf) != nw {
err = ErrShortWrite
return
}
if end {
break
}
}
return
}
// NewChunker returns a [ChunkLocker] ready to use.
func NewChunker(chunkSize uint) (c *ChunkLocker, err error) {
c = &ChunkLocker{}
err = c.SetChunkLen(chunkSize)
return
}
// NewCtxIO returns a [CtxIO].
func NewCtxIO(ctx context.Context, r io.Reader, w io.Writer, chunkSize uint) (c *CtxIO, err error) {
if r == nil {
err = ErrNilReader
return
}
if w == nil {
err = ErrNilWriter
return
}
if chunkSize == 0 {
err = ErrInvalidChunkSize
return
}
if ctx == nil {
err = ErrNilCtx
return
}
c = &CtxIO{
r: r,
w: w,
l: ChunkLocker{
chunkLen: chunkSize,
},
ctx: ctx,
}
return
}
/*
NewXIO returns a nil [XIO].
A weird "feature" of Golang is that a nil XIO is perfectly fine to use;
it's completely stateless and only has pointer receivers that only work with passed in
values so `new(XIO)` is completely unnecessary (as is NewXCopier).
In other words, this works fine:
var xc *iox.XIO
if n, err = xc.Copy(w, r); err != nil {
return
}
This function is just to maintain cleaner-looking code if you should so need it,
or want an XIO without declaring one:
if n, err = iox.NewXCopier().Copy(w, r); err != nil {
return
}
*/
func NewXIO() (x *XIO) {
// No-op lel
return
}

28
iox/funcs_chunklocker.go Normal file
View File

@@ -0,0 +1,28 @@
package iox
// GetChunkLen returns the current chunk size/length in bytes.
func (c *ChunkLocker) GetChunkLen() (size uint) {
c.lock.RLock()
defer c.lock.RUnlock()
size = c.chunkLen
return
}
// SetChunkLen sets the current chunk size/length in bytes.
func (c *ChunkLocker) SetChunkLen(size uint) (err error) {
if size == 0 {
err = ErrInvalidChunkSize
return
}
c.lock.Lock()
defer c.lock.Unlock()
c.chunkLen = size
return
}

173
iox/funcs_ctxio.go Normal file
View File

@@ -0,0 +1,173 @@
package iox
import (
`bytes`
`context`
`io`
`math`
)
func (c *CtxIO) Copy(dst io.Writer, src io.Reader) (written int64, err error) {
if c.l.chunkLen > math.MaxInt64 {
err = ErrChunkTooBig
}
return CopyCtxBufN(c.ctx, dst, src, int64(c.l.chunkLen))
}
func (c *CtxIO) CopyBufN(dst io.Writer, src io.Reader, n int64) (written int64, err error) {
if n <= 0 {
err = ErrBufTooSmall
return
}
return CopyCtxBufN(c.ctx, dst, src, n)
}
func (c *CtxIO) GetChunkLen() (size uint) {
return c.l.GetChunkLen()
}
func (c *CtxIO) Read(p []byte) (n int, err error) {
var nr int64
if nr, err = c.ReadWithContext(c.ctx, p); err != nil {
if nr > math.MaxInt {
n = math.MaxInt
} else {
n = int(nr)
}
return
}
if nr > math.MaxInt {
n = math.MaxInt
} else {
n = int(nr)
}
return
}
func (c *CtxIO) ReadWithContext(ctx context.Context, p []byte) (n int64, err error) {
var nr int
var off int
var buf []byte
if p == nil || len(p) == 0 {
return
}
if c.buf.Len() == 0 {
err = io.EOF
return
}
if c.l.chunkLen > uint(len(p)) {
// Would normally be a single chunk, so one-shot it.
nr, err = c.buf.Read(p)
n = int64(nr)
return
}
// Chunk over it.
endRead:
for {
select {
case <-ctx.Done():
err = ctx.Err()
return
default:
/*
off(set) is the index of the *next position* to write to.
Therefore the last offset == len(p),
therefore:
* if off == len(p), "done" (return no error, do *not* read from buf)
* if off + c.l.chunkLen > len(p), buf should be len(p) - off instead
*/
if off == len(p) {
break endRead
}
if uint(off)+c.l.chunkLen > uint(len(p)) {
buf = make([]byte, len(p)-off)
} else {
buf = make([]byte, c.l.chunkLen)
}
nr, err = c.buf.Read(buf)
n += int64(nr)
if nr > 0 {
off += nr
copy(p[off:], buf[:nr])
}
if err == io.EOF {
break endRead
} else if err != nil {
return
}
}
}
return
}
func (c *CtxIO) SetChunkLen(size uint) (err error) {
return c.l.SetChunkLen(size)
}
func (c *CtxIO) SetContext(ctx context.Context) (err error) {
if ctx == nil {
err = ErrNilCtx
return
}
c.ctx = ctx
return
}
func (c *CtxIO) Write(p []byte) (n int, err error) {
var nw int64
if c.l.chunkLen > math.MaxInt64 {
err = ErrChunkTooBig
return
}
if nw, err = c.WriteNWithContext(c.ctx, p, int64(c.l.chunkLen)); err != nil {
if nw > math.MaxInt {
n = math.MaxInt
} else {
n = int(nw)
}
return
}
if nw > math.MaxInt {
n = math.MaxInt
} else {
n = int(nw)
}
return
}
func (c *CtxIO) WriteNWithContext(ctx context.Context, p []byte, n int64) (written int64, err error) {
return CopyCtxBufN(ctx, &c.buf, bytes.NewReader(p), n)
}
func (c *CtxIO) WriteRune(r rune) (n int, err error) {
// We don't even bother listening for the ctx.Done because it's a single rune.
n, err = c.buf.WriteRune(r)
return
}
func (c *CtxIO) WriteWithContext(ctx context.Context, p []byte) (n int64, err error) {
if c.l.chunkLen > math.MaxInt64 {
err = ErrChunkTooBig
return
}
return CopyCtxBufN(ctx, &c.buf, bytes.NewReader(p), int64(c.l.chunkLen))
}

40
iox/funcs_xio.go Normal file
View File

@@ -0,0 +1,40 @@
package iox
import (
`io`
)
// Copy copies [io.Reader] `src` to [io.Writer] `dst`. It implements [Copier].
func (x *XIO) Copy(dst io.Writer, src io.Reader) (written int64, err error) {
return io.Copy(dst, src)
}
// CopyBuffer copies [io.Reader] `src` to [io.Writer] `dst` using buffer `buf`. It implements [CopyBufferer].
func (x *XIO) CopyBuffer(dst io.Writer, src io.Reader, buf []byte) (written int64, err error) {
return io.CopyBuffer(dst, src, buf)
}
// CopyBufWith copies [io.Reader] `src` to [io.Writer] `dst` using buffer returner `bufFunc`. It implements [SizedCopyBufferInvoker].
func (x *XIO) CopyBufWith(dst io.Writer, src io.Reader, bufFunc func() (b []byte)) (written int64, err error) {
return CopyBufWith(dst, src, bufFunc)
}
// CopyBufWithDynamic copies [io.Reader] `src` to [io.Writer] `dst` using buffer returner `bufFunc` for each chunk. It implements [DynamicSizedCopyBufferInvoker].
func (x *XIO) CopyBufWithDynamic(dst io.Writer, src io.Reader, bufFunc func() (b []byte)) (written int64, err error) {
return CopyBufWithDynamic(dst, src, bufFunc)
}
/*
CopyBufN reads buffered bytes from [io.Reader] `src` and copies to [io.Writer] `dst`
using the synchronous buffer size `n`.
It implements [SizedCopyBufferer].
*/
func (x *XIO) CopyBufN(dst io.Writer, src io.Reader, n int64) (written int64, err error) {
return CopyBufN(dst, src, n)
}
// CopyN copies from [io.Reader] `src` to [io.Writer] `w`, `n` bytes at a time. It implements [SizedCopier].
func (x *XIO) CopyN(dst io.Writer, src io.Reader, n int64) (written int64, err error) {
return io.CopyN(dst, src, n)
}

View File

@@ -1,8 +1,209 @@
package iox package iox
import (
`bytes`
`context`
`io`
`sync`
)
type ( type (
// RuneWriter matches the behavior of *(bytes.Buffer).WriteRune and *(bufio.Writer).WriteRune /*
RuneWriter matches the behavior of [bytes.Buffer.WriteRune] and [bufio.Writer.WriteRune].
(Note that this package does not have a "RuneReader"; see [io.RuneReader] instead.)
*/
RuneWriter interface { RuneWriter interface {
WriteRune(r rune) (n int, err error) WriteRune(r rune) (n int, err error)
} }
// Copier matches the signature/behavior of [io.Copy]. Implemented by [XIO].
Copier interface {
Copy(dst io.Writer, src io.Reader) (written int64, err error)
}
// CopyBufferer matches the signature/behavior of [io.CopyBuffer]. Implemented by [XIO].
CopyBufferer interface {
CopyBuffer(dst io.Writer, src io.Reader, buf []byte) (written int64, err error)
}
// SizedCopier matches the signature/behavior of [io.CopyN]. Implemented by [XIO].
SizedCopier interface {
CopyN(dst io.Writer, src io.Reader, n int64) (written int64, err error)
}
// SizedCopyBufferer matches the signature/behavior of [CopyBufN]. Implemented by [XIO].
SizedCopyBufferer interface {
CopyBufN(dst io.Writer, src io.Reader, n int64) (written int64, err error)
}
// SizedCopyBufferInvoker matches the signature/behavior of [CopyBufWith]. Implemented by [XIO].
SizedCopyBufferInvoker interface {
CopyBufWith(dst io.Writer, src io.Reader, bufFunc func() (b []byte)) (written int64, err error)
}
// DynamicSizedCopyBufferInvoker matches the signature/behavior of [CopyBufWithDynamic]. Implemented by [XIO].
DynamicSizedCopyBufferInvoker interface {
CopyBufWithDynamic(dst io.Writer, src io.Reader, bufFunc func() (b []byte)) (written int64, err error)
}
/*
Chunker is used by both [ContextReader] and [ContextWriter] to set/get the current chunk size.
Chunking is inherently required to be specified in order to interrupt reads/writes/copies with a [context.Context].
Implementations *must* use a [sync.RWMutex] to get (RLock) and set (Lock) the chunk size.
The chunk size *must not* be directly accessible to maintain concurrency safety assumptions.
*/
Chunker interface {
// GetChunkLen returns the current chunk size/length in bytes.
GetChunkLen() (size uint)
// SetChunkLen sets the current chunk size/length in bytes.
SetChunkLen(size uint) (err error)
}
/*
ChunkReader implements a chunking reader.
Third-party implementations *must* respect the chunk size locking (see [Chunker]).
The Read method should read in chunks of the internal chunk size.
*/
ChunkReader interface {
io.Reader
Chunker
}
/*
ChunkWriter implements a chunking writer.
Third-party implementations *must* respect the chunk size locking (see [Chunker]).
The Write method should write out in chunks of the internal chunk size.
*/
ChunkWriter interface {
io.Writer
Chunker
}
// ChunkReadWriter implements a chunking reader/writer.
ChunkReadWriter interface {
ChunkReader
ChunkWriter
}
/*
ContextSetter allows one to set an internal context.
A nil context should return an error.
*/
ContextSetter interface {
SetContext(context context.Context) (err error)
}
/*
ContextCopier is defined to allow for consumer-provided types. See [CtxIO] for a package-provided type.
The Copy method should use an internal context and chunk size
(and thus wrap [CopyCtxBufN] internally on an external call to Copy, etc.).
*/
ContextCopier interface {
Copier
Chunker
ContextSetter
SizedCopyBufferer
}
/*
ContextReader is primarily here to allow for consumer-provided types. See [CtxIO] for a package-provided type.
The Read method should use an internal context and chunk size.
The ReadWithContext method should use an internal chunk size.
*/
ContextReader interface {
ChunkReader
ContextSetter
ReadWithContext(ctx context.Context, p []byte) (n int64, err error)
}
/*
ContextWriter is primarily here to allow for consumer-provided types. See [CtxIO] for a package-provided type.
The Write method should use an internal context.
The WriteWithContext should use an internal chunk size.
*/
ContextWriter interface {
ChunkWriter
ContextSetter
WriteWithContext(ctx context.Context, p []byte) (n int64, err error)
WriteNWithContext(ctx context.Context, p []byte, n int64) (written int64, err error)
}
/*
ContextReadWriter is primarily here to allow for consumer-provided types.
See [CtxIO] for a package-provided type.
*/
ContextReadWriter interface {
ContextReader
ContextWriter
}
)
type (
// ChunkLocker implements [Chunker].
ChunkLocker struct {
lock sync.RWMutex
chunkLen uint
}
/*
CtxIO is a type used to demonstrate "stateful" I/O introduced by this package.
It implements:
* [Copier]
* [Chunker]
* [RuneWriter]
* [ChunkReader]
* [ChunkWriter]
* [ContextCopier]
* [ContextSetter]
* [ContextReader]
* [ContextWriter]
* [ChunkReadWriter]
* [ContextReadWriter]
* [SizedCopyBufferer]
Unlike [XIO], it must be non-nil (see [NewCtxIO]) since it maintains state
(though technically, one does not need to call [NewCtxIO] if they call
[CtxIO.SetChunkLen] and [CtxIO.SetContext] before any other methods).
[CtxIO.Read] and other Read methods writes to an internal buffer,
and [CtxIO.Write] and other Write methods writes out from it.
*/
CtxIO struct {
r io.Reader
w io.Writer
l ChunkLocker
buf bytes.Buffer
ctx context.Context
}
/*
XIO is a type used to demonstrate "stateless" I/O introduced by this package.
It implements:
* [Copier]
* [CopyBufferer]
* [SizedCopier]
* [SizedCopyBufferer]
* [SizedCopyBufferInvoker]
* [DynamicSizedCopyBufferInvoker]
Unlike [CtxIO], the zero-value is ready to use since it holds no state
or configuration whatsoever.
A nil XIO is perfectly usable but if you want something more idiomatic,
see [NewXIO].
*/
XIO struct{}
) )

4
mapsx/doc.go Normal file
View File

@@ -0,0 +1,4 @@
/*
Package mapsx includes functions that probably should have been in [maps] but aren't.
*/
package mapsx

9
mapsx/errs.go Normal file
View File

@@ -0,0 +1,9 @@
package mapsx
import (
`errors`
)
var (
ErrNotFound = errors.New("key not found")
)

43
mapsx/funcs.go Normal file
View File

@@ -0,0 +1,43 @@
package mapsx
/*
Get mimics Python's [dict.get()] behavior, returning value `v` if key `k`
is not found in map `m`.
See also [GetOk], [Must].
[dict.get()]: https://docs.python.org/3/library/stdtypes.html#dict.get
*/
func Get[Map ~map[K]V, K comparable, V any](m Map, k K, v V) (val V) {
val, _ = GetOk(m, k, v)
return
}
// GetOk is like [Get] but also explicitly indicates whether `k` was found or not. See also [Must].
func GetOk[Map ~map[K]V, K comparable, V any](m Map, k K, v V) (val V, found bool) {
if val, found = m[k]; !found {
val = v
}
return
}
/*
Must, unlike [Get] or [GetOk], requires that `k` be in map `m`.
A panic with error [ErrNotFound] will be raised if `k` is not present.
Otherwise the found value will be returned.
*/
func Must[Map ~map[K]V, K comparable, V any](m Map, k K) (val V) {
var ok bool
if val, ok = m[k]; !ok {
panic(ErrNotFound)
}
return
}

View File

@@ -10,11 +10,20 @@ const (
) )
const ( const (
// cksumMask is AND'd with a checksum to get the "carried ones". /*
cksumMask is AND'd with a checksum to get the "carried ones"
(the lower 16 bits before folding carries).
*/
cksumMask uint32 = 0x0000ffff cksumMask uint32 = 0x0000ffff
// cksumShift is used in the "carried-ones folding". /*
cksumShift is used in the "carried-ones folding";
it's the number of bits to right-shift the carry-over.
*/
cksumShift uint32 = 0x00000010 cksumShift uint32 = 0x00000010
// padShift is used to "pad out" a checksum for odd-length buffers by left-shifting. /*
padShift is used to "pad out" a checksum for odd-length buffers by left-shifting.
It positions the high-byte of a 16-byte "word" (big-endian, as per ord below).
*/
padShift uint32 = 0x00000008 padShift uint32 = 0x00000008
) )

View File

@@ -25,6 +25,9 @@ safety and no data retention, which can be used as a:
* [io.StringWriter] * [io.StringWriter]
* [io.Writer] * [io.Writer]
If you don't need all these interfaces, a reasonable alternative may be
to use gVisor's [gvisor.dev/gvisor/pkg/tcpip/checksum] instead.
[RFC 1071]: https://datatracker.ietf.org/doc/html/rfc1071 [RFC 1071]: https://datatracker.ietf.org/doc/html/rfc1071
[RFC 1141]: https://datatracker.ietf.org/doc/html/rfc1141 [RFC 1141]: https://datatracker.ietf.org/doc/html/rfc1141
[RFC 1624]: https://datatracker.ietf.org/doc/html/rfc1624 [RFC 1624]: https://datatracker.ietf.org/doc/html/rfc1624

View File

@@ -7,8 +7,9 @@ import (
// New returns a new initialized [InetChecksum]. It will never panic. // New returns a new initialized [InetChecksum]. It will never panic.
func New() (i *InetChecksum) { func New() (i *InetChecksum) {
i = &InetChecksum{} i = &InetChecksum{
_ = i.Aligned() aligned: true,
}
return return
} }
@@ -21,15 +22,14 @@ b may be nil or 0-length; this will not cause an error.
func NewFromBytes(b []byte) (i *InetChecksum, copied int, err error) { func NewFromBytes(b []byte) (i *InetChecksum, copied int, err error) {
var cksum InetChecksum var cksum InetChecksum
var cptr *InetChecksum = &cksum
cksum.aligned = true
if b != nil && len(b) > 0 { if b != nil && len(b) > 0 {
if copied, err = cksum.Write(b); err != nil { if copied, err = cptr.Write(b); err != nil {
return return
} }
_ = i.Aligned()
} else {
i = New()
return
} }
i = &cksum i = &cksum
@@ -48,7 +48,64 @@ func NewFromBuf(buf io.Reader) (i *InetChecksum, copied int64, err error) {
var cksum InetChecksum var cksum InetChecksum
_ = i.Aligned() cksum.aligned = true
if buf != nil {
if copied, err = io.Copy(&cksum, buf); err != nil {
return
}
}
i = &cksum
return
}
// NewSimple returns a new initialized [InetChecksumSimple]. It will never panic.
func NewSimple() (i *InetChecksumSimple) {
i = &InetChecksumSimple{
aligned: true,
}
return
}
/*
NewSimpleFromBytes returns a new [InetChecksumSimple] initialized with explicit bytes.
b may be nil or 0-length; this will not cause an error.
*/
func NewSimpleFromBytes(b []byte) (i *InetChecksumSimple, copied int, err error) {
var cksum InetChecksumSimple
var cptr *InetChecksumSimple = &cksum
cksum.aligned = true
if b != nil && len(b) > 0 {
if copied, err = cptr.Write(b); err != nil {
return
}
}
i = &cksum
return
}
/*
NewSimpleFromBuf returns an [InetChecksumSimple] from a specified [io.Reader].
buf may be nil. If it isn't, NewSimpleFromBuf will call [io.Copy] on buf.
Note that this may exhaust your passed buf or advance its current seek position/offset,
depending on its type.
*/
func NewSimpleFromBuf(buf io.Reader) (i *InetChecksumSimple, copied int64, err error) {
var cksum InetChecksumSimple
cksum.aligned = true
if buf != nil { if buf != nil {
if copied, err = io.Copy(&cksum, buf); err != nil { if copied, err = io.Copy(&cksum, buf); err != nil {

View File

@@ -22,7 +22,7 @@ func (i *InetChecksum) Aligned() (aligned bool) {
defer i.alignLock.Unlock() defer i.alignLock.Unlock()
i.bufLock.RLock() i.bufLock.RLock()
aligned = i.buf.Len()&2 == 0 aligned = i.buf.Len()%2 == 0
i.bufLock.RUnlock() i.bufLock.RUnlock()
i.aligned = aligned i.aligned = aligned
@@ -113,7 +113,7 @@ func (i *InetChecksum) Reset() {
i.sumLock.Lock() i.sumLock.Lock()
i.lastLock.Lock() i.lastLock.Lock()
i.aligned = false i.aligned = true
i.alignLock.Unlock() i.alignLock.Unlock()
i.buf.Reset() i.buf.Reset()
@@ -308,7 +308,7 @@ func (i *InetChecksum) WriteByte(c byte) (err error) {
} }
if !i.disabledBuf { if !i.disabledBuf {
if err = i.WriteByte(c); err != nil { if err = i.buf.WriteByte(c); err != nil {
i.sum = origSum i.sum = origSum
i.aligned = origAligned i.aligned = origAligned
i.last = origLast i.last = origLast

View File

@@ -27,7 +27,7 @@ func (i *InetChecksumSimple) Reset() {
i.last = 0x00 i.last = 0x00
i.sum = 0 i.sum = 0
i.last = 0x00 i.aligned = true
} }

View File

@@ -17,8 +17,8 @@ type (
If [InetChecksum.Aligned] returns false, the checksum result of an If [InetChecksum.Aligned] returns false, the checksum result of an
[InetChecksum.Sum] or [InetChecksum.Sum16] (or any other operation [InetChecksum.Sum] or [InetChecksum.Sum16] (or any other operation
returning a sum) will INCLUDE THE PAD NULL BYTE (which is only returning a sum) will INCLUDE THE PAD NULL BYTE (which is only
applied *at the time of the Sum/Sum32 call) and is NOT applied to applied *at the time of the Sum/Sum32 call* and is NOT applied to
the persistent underlying storage. the persistent underlying storage).
InetChecksum differs from [InetChecksumSimple] in that it: InetChecksum differs from [InetChecksumSimple] in that it:

View File

@@ -1,4 +1,12 @@
/* /*
Package remap provides convenience functions around regular expressions, primarily offering maps for named capture groups. Package remap provides convenience functions around regular expressions,
primarily offering maps for named capture groups.
It offers convenience equivalents of the following:
* [regexp.Compile] ([Compile])
* [regexp.CompilePOSIX] ([CompilePOSIX])
* [regexp.MustCompile] ([MustCompile])
* [regexp.MustCompilePOSIX] ([MustCompilePOSIX])
*/ */
package remap package remap

11
remap/errs.go Normal file
View File

@@ -0,0 +1,11 @@
package remap
import (
`errors`
)
var (
ErrInvalidIdxPair error = errors.New("invalid index pair; [1] must be >= [0]")
ErrNoStr error = errors.New("no string to slice/reslice/subslice")
ErrShortStr error = errors.New("string too short to slice/reslice/subslice")
)

170
remap/funcs.go Normal file
View File

@@ -0,0 +1,170 @@
package remap
import (
"regexp"
)
/*
Compile is a convenience shorthand for:
var err error
var r *remap.ReMap = new(remap.ReMap)
if r.Regexp, err = regexp.Compile(expr); err != nil {
// ...
}
It corresponds to [regexp.Compile].
*/
func Compile(expr string) (r *ReMap, err error) {
var p *regexp.Regexp
if p, err = regexp.Compile(expr); err != nil {
return
}
r = &ReMap{
Regexp: p,
}
return
}
/*
CompilePOSIX is a convenience shorthand for:
var err error
var r *remap.ReMap = new(remap.ReMap)
if r.Regexp, err = regexp.CompilePOSIX(expr); err != nil {
// ...
}
It corresponds to [regexp.CompilePOSIX].
*/
func CompilePOSIX(expr string) (r *ReMap, err error) {
var p *regexp.Regexp
if p, err = regexp.CompilePOSIX(expr); err != nil {
return
}
r = &ReMap{
Regexp: p,
}
return
}
/*
MustCompile is a convenience shorthand for:
var r *remap.ReMap = &remap.ReMap{
Regexp: regexp.MustCompile(expr),
}
It corresponds to [regexp.MustCompile].
*/
func MustCompile(expr string) (r *ReMap) {
var err error
var p *regexp.Regexp
// We panic ourselves instead of wrapping regexp.MustCompile.
// Makes debuggers a little more explicit.
if p, err = regexp.Compile(expr); err != nil {
panic(err)
}
r = &ReMap{
Regexp: p,
}
return
}
/*
MustCompilePOSIX is a convenience shorthand for:
var r *remap.ReMap = &remap.ReMap{
Regexp: regexp.MustCompilePOSIX(expr),
}
It corresponds to [regexp.MustCompilePOSIX].
*/
func MustCompilePOSIX(expr string) (r *ReMap) {
var err error
var p *regexp.Regexp
// We panic ourselves instead of wrapping regexp.MustCompilePOSIX.
// Makes debuggers a little more explicit.
if p, err = regexp.CompilePOSIX(expr); err != nil {
panic(err)
}
r = &ReMap{
Regexp: p,
}
return
}
/*
strIdxSlicer takes string s, and returns the substring marked by idxPair,
where:
idxPair = [2]int{
<substring START POSITION>,
<substring END BOUNDARY>,
}
That is, to get `oo` from `foobar`,
idxPair = [2]int{1, 3}
# NOT:
#idxPair = [2]int{1, 2}
subStr will be empty and matched will be false if:
* idxPair[0] < 0
* idxPair[1] < 0
It will panic with [ErrShortStr] if:
* idxPair[0] > len(s)-1
* idxPair[1] > len(s)
It will panic with [ErrInvalidIdxPair] if:
* idxPair[0] > idxPair[1]
It will properly handle single-character addresses (i.e. idxPair[0] == idxPair[1]).
*/
func strIdxSlicer(s string, idxPair [2]int) (subStr string, matched bool) {
if idxPair[0] < 0 || idxPair[1] < 0 {
return
}
matched = true
if (idxPair[0] > (len(s) - 1)) ||
(idxPair[1] > len(s)) {
panic(ErrShortStr)
}
if idxPair[0] > idxPair[1] {
panic(ErrInvalidIdxPair)
}
if idxPair[0] == idxPair[1] {
// single character
subStr = string(s[idxPair[0]])
} else {
// multiple characters
subStr = s[idxPair[0]:idxPair[1]]
}
return
}

View File

@@ -5,9 +5,14 @@ Map returns a map[string][]<match bytes> for regexes with named capture groups m
Note that this supports non-unique group names; [regexp.Regexp] allows for patterns with multiple groups Note that this supports non-unique group names; [regexp.Regexp] allows for patterns with multiple groups
using the same group name (though your IDE might complain; I know GoLand does). using the same group name (though your IDE might complain; I know GoLand does).
It will panic if the embedded [regexp.Regexp] is nil.
Each match for each group is in a slice keyed under that group name, with that slice Each match for each group is in a slice keyed under that group name, with that slice
ordered by the indexing done by the regex match itself. ordered by the indexing done by the regex match itself.
This operates on only the first found match (like [regexp.Regexp.FindSubmatch]).
To operate on *all* matches, use [ReMap.MapAll].
In summary, the parameters are as follows: In summary, the parameters are as follows:
# inclNoMatch # inclNoMatch
@@ -31,6 +36,7 @@ is provided but b does not match then matches will be:
If true (and inclNoMatch is true), instead of a single nil the group's values will be If true (and inclNoMatch is true), instead of a single nil the group's values will be
a slice of nil values explicitly matching the number of times the group name is specified a slice of nil values explicitly matching the number of times the group name is specified
in the pattern. in the pattern.
May be unpredictable if the same name is used multiple times for different capture groups across multiple patterns.
For example, if a pattern: For example, if a pattern:
@@ -87,7 +93,7 @@ In detail, matches and/or its values may be nil or empty under the following con
IF inclNoMatch is true IF inclNoMatch is true
IF inclNoMatchStrict is true IF inclNoMatchStrict is true
THEN matches[<group name>] is defined and non-nil, but populated with placeholder nils THEN matches[<group name>] is defined and non-nil, but populated with placeholder nils
(matches[<group name>] == [][]byte{nil[, nil...]}) (matches[<group name>] == [][]byte{nil[, nil, ...]})
ELSE ELSE
THEN matches[<group name>] is guaranteed defined but may be nil (_, ok = matches[<group name>]; ok == true) THEN matches[<group name>] is guaranteed defined but may be nil (_, ok = matches[<group name>]; ok == true)
ELSE ELSE
@@ -109,7 +115,7 @@ func (r *ReMap) Map(b []byte, inclNoMatch, inclNoMatchStrict, mustMatch bool) (m
return return
} }
names = r.Regexp.SubexpNames() names = r.Regexp.SubexpNames()[:]
matchBytes = r.Regexp.FindSubmatch(b) matchBytes = r.Regexp.FindSubmatch(b)
if matchBytes == nil { if matchBytes == nil {
@@ -142,6 +148,9 @@ func (r *ReMap) Map(b []byte, inclNoMatch, inclNoMatchStrict, mustMatch bool) (m
if inclNoMatch { if inclNoMatch {
if len(names) >= 1 { if len(names) >= 1 {
for _, grpNm = range names { for _, grpNm = range names {
if grpNm == "" {
continue
}
matches[grpNm] = nil matches[grpNm] = nil
} }
} }
@@ -154,7 +163,7 @@ func (r *ReMap) Map(b []byte, inclNoMatch, inclNoMatchStrict, mustMatch bool) (m
grpNm = names[mIdx] grpNm = names[mIdx]
/* /*
Thankfully, it's actually a build error if a pattern specifies a named Thankfully, it's actually a build error if a pattern specifies a named
capture group with an empty name. capture group with an matched name.
So we don't need to worry about accounting for that, So we don't need to worry about accounting for that,
and can just skip over grpNm == "" (which is an *unnamed* capture group). and can just skip over grpNm == "" (which is an *unnamed* capture group).
*/ */
@@ -190,6 +199,9 @@ func (r *ReMap) Map(b []byte, inclNoMatch, inclNoMatchStrict, mustMatch bool) (m
// This *technically* should be completely handled above. // This *technically* should be completely handled above.
if inclNoMatch { if inclNoMatch {
for _, grpNm = range names { for _, grpNm = range names {
if grpNm == "" {
continue
}
if _, ok = tmpMap[grpNm]; !ok { if _, ok = tmpMap[grpNm]; !ok {
tmpMap[grpNm] = nil tmpMap[grpNm] = nil
} }
@@ -204,13 +216,147 @@ func (r *ReMap) Map(b []byte, inclNoMatch, inclNoMatchStrict, mustMatch bool) (m
} }
/* /*
MapString is exactly like ReMap.Map(), but operates on (and returns) strings instead. MapAll behaves exactly like [ReMap.Map] but will "squash"/consolidate *all* found matches, not just the first occurrence,
(matches will always be nil if s == “.) into the group name.
A small deviation, though; empty strings instead of nils (because duh) will occupy slice placeholders (if `inclNoMatchStrict` is specified). You likely want to use this instead of [ReMap.Map] for multiline patterns.
*/
func (r *ReMap) MapAll(b []byte, inclNoMatch, inclNoMatchStrict, mustMatch bool) (matches map[string][][]byte) {
var ok bool
var mIdx int
var isEmpty bool
var match []byte
var grpNm string
var names []string
var mbGrp [][]byte
var ptrnNms []string
var matchBytes [][][]byte
var tmpMap map[string][][]byte = make(map[string][][]byte)
if b == nil {
return
}
names = r.Regexp.SubexpNames()[:]
matchBytes = r.Regexp.FindAllSubmatch(b, -1)
if matchBytes == nil {
// b does not match pattern
if !mustMatch {
matches = make(map[string][][]byte)
}
return
}
if names == nil || len(names) == 0 || len(names) == 1 {
/*
no named capture groups;
technically only the last condition would be the case.
*/
if inclNoMatch {
matches = make(map[string][][]byte)
}
return
}
names = names[1:]
tmpMap = make(map[string][][]byte)
// From here, it behaves (sort of) like ReMap.Map
// except mbGrp is like matchBytes in Map.
for _, mbGrp = range matchBytes {
// Unlike ReMap.Map, we have to do a little additional logic.
isEmpty = false
ptrnNms = make([]string, 0, len(names))
if mbGrp == nil {
isEmpty = true
}
if !isEmpty {
if len(mbGrp) == 0 || len(mbGrp) == 1 {
/*
no submatches whatsoever.
*/
isEmpty = true
} else {
mbGrp = mbGrp[1:]
for mIdx, match = range mbGrp {
if mIdx > len(names) {
break
}
grpNm = names[mIdx]
if grpNm == "" {
continue
}
ptrnNms = append(ptrnNms, grpNm)
if match == nil {
// This specific group didn't match, but it matched the whole pattern.
if !inclNoMatch {
continue
}
if _, ok = tmpMap[grpNm]; !ok {
if !inclNoMatchStrict {
tmpMap[grpNm] = nil
} else {
tmpMap[grpNm] = [][]byte{nil}
}
} else {
if inclNoMatchStrict {
tmpMap[grpNm] = append(tmpMap[grpNm], nil)
}
}
continue
}
if _, ok = tmpMap[grpNm]; !ok {
tmpMap[grpNm] = make([][]byte, 0)
}
tmpMap[grpNm] = append(tmpMap[grpNm], match)
}
}
}
// I can't recall why I capture this.
_ = ptrnNms
}
// *Theoretically* all of these should be populated with at least a nil.
if inclNoMatch {
for _, grpNm = range names {
if grpNm == "" {
continue
}
if _, ok = tmpMap[grpNm]; !ok {
tmpMap[grpNm] = nil
}
}
}
if len(tmpMap) > 0 {
matches = tmpMap
}
return
}
/*
MapString is exactly like [ReMap.Map], but operates on (and returns) strings instead.
(matches will always be nil if s == "".)
It will panic if the embedded [regexp.Regexp] is nil.
This operates on only the first found match (like [regexp.Regexp.FindStringSubmatch]).
To operate on *all* matches, use [ReMap.MapStringAll].
A small deviation and caveat, though; empty strings instead of nils (because duh) will occupy slice placeholders (if `inclNoMatchStrict` is specified).
This unfortunately *does not provide any indication* if an empty string positively matched the pattern (a "hit") or if it was simply This unfortunately *does not provide any indication* if an empty string positively matched the pattern (a "hit") or if it was simply
not matched at all (a "miss"). If you need definitive determination between the two conditions, it is instead recommended to either not matched at all (a "miss"). If you need definitive determination between the two conditions, it is instead recommended to either
*not* use inclNoMatchStrict or to use ReMap.Map() instead and convert any non-nil values to strings after. *not* use inclNoMatchStrict or to use [ReMap.Map] instead and convert any non-nil values to strings after.
Particularly: Particularly:
@@ -233,8 +379,9 @@ is provided but s does not match then matches will be:
# inclNoMatchStrict # inclNoMatchStrict
If true (and inclNoMatch is true), instead of a single nil the group's values will be If true (and inclNoMatch is true), instead of a single nil the group's values will be
a slice of eempty string values explicitly matching the number of times the group name is specified a slice of empty string values explicitly matching the number of times the group name is specified
in the pattern. in the pattern.
May be unpredictable if the same name is used multiple times for different capture groups across multiple patterns.
For example, if a pattern: For example, if a pattern:
@@ -290,8 +437,8 @@ In detail, matches and/or its values may be nil or empty under the following con
IF <group name> does not have a match IF <group name> does not have a match
IF inclNoMatch is true IF inclNoMatch is true
IF inclNoMatchStrict is true IF inclNoMatchStrict is true
THEN matches[<group name>] is defined and non-nil, but populated with placeholder nils THEN matches[<group name>] is defined and non-nil, but populated with placeholder strings
(matches[<group name>] == []string{""[, ""...]}) (matches[<group name>] == []string{""[, "", ...]})
ELSE ELSE
THEN matches[<group name>] is guaranteed defined but may be nil (_, ok = matches[<group name>]; ok == true) THEN matches[<group name>] is guaranteed defined but may be nil (_, ok = matches[<group name>]; ok == true)
ELSE ELSE
@@ -304,27 +451,19 @@ func (r *ReMap) MapString(s string, inclNoMatch, inclNoMatchStrict, mustMatch bo
var ok bool var ok bool
var endIdx int var endIdx int
var startIdx int var startIdx int
var chunkIdx int var grpIdx int
var grpNm string var grpNm string
var names []string var names []string
var matchStr string var matchStr string
/* var si stringIndexer
A slice of indices or index pairs.
For each element `e` in idxChunks,
* if `e` is nil, no group match.
* if len(e) == 1, only a single character was matched.
* otherwise len(e) == 2, the start and end of the match.
*/
var idxChunks [][]int
var matchIndices []int var matchIndices []int
var chunkIndices []int // always 2 elements; start pos and end pos
var tmpMap map[string][]string = make(map[string][]string) var tmpMap map[string][]string = make(map[string][]string)
/* /*
OK so this is a bit of a deviation. OK so this is a bit of a deviation.
It's not as straightforward as above, because there isn't an explicit way It's not as straightforward as above, because there isn't an explicit way
like above to determine if a pattern was *matched as an empty string* vs. like above to determine if a pattern was *matched as an matched string* vs.
*not matched*. *not matched*.
So instead do roundabout index-y things. So instead do roundabout index-y things.
@@ -334,7 +473,8 @@ func (r *ReMap) MapString(s string, inclNoMatch, inclNoMatchStrict, mustMatch bo
return return
} }
/* /*
I'm not entirely sure how serious they are about "the slice should not be modified"... I'm not entirely sure how serious they are about
"the slice should not be modified"...
DO NOT sort or dedupe `names`! If the same name for groups is duplicated, DO NOT sort or dedupe `names`! If the same name for groups is duplicated,
it will be duplicated here in proper order and the ordering is tied to it will be duplicated here in proper order and the ordering is tied to
@@ -351,7 +491,7 @@ func (r *ReMap) MapString(s string, inclNoMatch, inclNoMatchStrict, mustMatch bo
return return
} }
if names == nil || len(names) <= 1 { if names == nil || len(names) == 0 || len(names) == 1 {
/* /*
No named capture groups; No named capture groups;
technically only the last condition would be the case, technically only the last condition would be the case,
@@ -363,6 +503,7 @@ func (r *ReMap) MapString(s string, inclNoMatch, inclNoMatchStrict, mustMatch bo
} }
return return
} }
names = names[1:]
if len(matchIndices) == 0 || len(matchIndices) == 1 { if len(matchIndices) == 0 || len(matchIndices) == 1 {
/* /*
@@ -378,26 +519,34 @@ func (r *ReMap) MapString(s string, inclNoMatch, inclNoMatchStrict, mustMatch bo
matches = make(map[string][]string) matches = make(map[string][]string)
if inclNoMatch { if inclNoMatch {
for _, grpNm = range names { for _, grpNm = range names {
if grpNm != "" { if grpNm == "" {
matches[grpNm] = nil continue
} }
matches[grpNm] = nil
} }
} }
return return
} }
/* /*
A reslice of `matchIndices` could technically start at 2 (as long as `names` is sliced [1:]) The reslice of `matchIndices` starts at 2 because they're in pairs:
because they're in pairs: []int{<start>, <end>, <start>, <end>, ...}
and the first pair is the entire pattern match (un-resliced names[0]). []int{<start>, <end>, <start>, <end>, ...}
Thus the len(matchIndices) == 2*len(names), *even* if you
and the first pair is the entire pattern match (un-resliced names[0],
un-resliced matchIndices[0]).
Thus the len(matchIndices) == 2*len(names) (*should*, that is), *even* if you reslice.
Keep in mind that since the first element of names is removed, Keep in mind that since the first element of names is removed,
the first pair here is skipped. we reslice matchIndices as well.
This provides a bit more consistent readability, though.
*/ */
idxChunks = make([][]int, len(names)) matchIndices = matchIndices[2:]
chunkIdx = 0
endIdx = 0 tmpMap = make(map[string][]string)
// Note that the second index is the *upper boundary*, not a *position in the string*
// so these indices are perfectly usable as-is as returned from the regexp methods.
// http://golang.org/ref/spec#Slice_expressions
for startIdx = 0; endIdx < len(matchIndices); startIdx += 2 { for startIdx = 0; endIdx < len(matchIndices); startIdx += 2 {
endIdx = startIdx + 2 endIdx = startIdx + 2
// This technically should never happen. // This technically should never happen.
@@ -405,75 +554,253 @@ func (r *ReMap) MapString(s string, inclNoMatch, inclNoMatchStrict, mustMatch bo
endIdx = len(matchIndices) endIdx = len(matchIndices)
} }
chunkIndices = matchIndices[startIdx:endIdx] if grpIdx >= len(names) {
break
if chunkIndices[0] == -1 || chunkIndices[1] == -1 {
// group did not match
chunkIndices = nil
} else {
if chunkIndices[0] == chunkIndices[1] {
chunkIndices = []int{chunkIndices[0]}
} else {
chunkIndices = matchIndices[startIdx:endIdx]
}
} }
idxChunks[chunkIdx] = chunkIndices
chunkIdx++
}
// Now associate with names and pull the string sequence. si = stringIndexer{
for chunkIdx, chunkIndices = range idxChunks { group: grpIdx,
grpNm = names[chunkIdx] start: matchIndices[startIdx],
/* end: matchIndices[endIdx-1],
Thankfully, it's actually a build error if a pattern specifies a named matched: true,
capture group with an empty name. nm: names[grpIdx],
So we don't need to worry about accounting for that, grpS: "",
and can just skip over grpNm == "" s: &matchStr,
(which is either an *unnamed* capture group ptrn: r.Regexp,
OR the first element in `names`, which is always }
the entire match). grpIdx++
*/
if grpNm == "" { if si.nm == "" {
// unnamed capture group
continue continue
} }
if chunkIndices == nil || len(chunkIndices) == 0 { // sets si.matched and si.grpS
// group did not match si.idxSlice(&s)
if !si.matched {
if !inclNoMatch { if !inclNoMatch {
continue continue
} }
if _, ok = tmpMap[grpNm]; !ok { if _, ok = tmpMap[si.nm]; !ok {
if !inclNoMatchStrict { if !inclNoMatchStrict {
tmpMap[grpNm] = nil tmpMap[si.nm] = nil
} else { } else {
tmpMap[grpNm] = []string{""} tmpMap[si.nm] = []string{""}
} }
} else { } else {
if inclNoMatchStrict { if inclNoMatchStrict {
tmpMap[grpNm] = append(tmpMap[grpNm], "") tmpMap[si.nm] = append(tmpMap[si.nm], "")
} }
} }
continue continue
} }
switch len(chunkIndices) { if _, ok = tmpMap[si.nm]; !ok {
case 1: tmpMap[si.nm] = make([]string, 0)
// Single character
matchStr = string(s[chunkIndices[0]])
case 2:
// Multiple characters
matchStr = s[chunkIndices[0]:chunkIndices[1]]
} }
tmpMap[si.nm] = append(tmpMap[si.nm], si.grpS)
if _, ok = tmpMap[grpNm]; !ok {
tmpMap[grpNm] = make([]string, 0)
}
tmpMap[grpNm] = append(tmpMap[grpNm], matchStr)
} }
// This *technically* should be completely handled above. // This *technically* should be completely handled above.
if inclNoMatch { if inclNoMatch {
for _, grpNm = range names { for _, grpNm = range names {
if grpNm == "" {
continue
}
if _, ok = tmpMap[grpNm]; !ok {
tmpMap[grpNm] = nil
}
}
}
if len(tmpMap) > 0 {
matches = tmpMap
}
return
}
/*
MapStringAll behaves exactly like [ReMap.MapString] but will "squash"/consolidate *all* found matches, not just the first occurrence,
into the group name.
You likely want to use this instead of [ReMap.MapString] for multiline patterns.
*/
func (r *ReMap) MapStringAll(s string, inclNoMatch, inclNoMatchStrict, mustMatch bool) (matches map[string][]string) {
var ok bool
var endIdx int
var startIdx int
var grpIdx int
var grpNm string
var names []string
var matchStr string
var si stringIndexer
var matchIndices []int
var allMatchIndices [][]int
var tmpMap map[string][]string = make(map[string][]string)
if s == "" {
return
}
names = r.Regexp.SubexpNames()[:]
allMatchIndices = r.Regexp.FindAllStringSubmatchIndex(s, -1)
if allMatchIndices == nil {
// s does not match pattern at all.
if !mustMatch {
matches = make(map[string][]string)
}
return
}
if names == nil || len(names) == 0 || len(names) == 1 {
/*
No named capture groups;
technically only the last condition would be the case,
as (regexp.Regexp).SubexpNames() will ALWAYS at the LEAST
return a `[]string{""}`.
*/
if inclNoMatch {
matches = make(map[string][]string)
}
return
}
names = names[1:]
if len(allMatchIndices) == 0 {
// No matches (and thus submatches) whatsoever.
// I think this is actually covered by the `if allMatchIndices == nil { ... }` above,
// but this is still here for safety and efficiency - early return on no matches to iterate.
matches = make(map[string][]string)
if inclNoMatch {
for _, grpNm = range names {
if grpNm == "" {
continue
}
matches[grpNm] = nil
}
}
return
}
// Do *NOT* trim/reslice allMatchIndices!
// The reslicing is done below, *inside* each matchIndices iteration!
tmpMap = make(map[string][]string)
// From here, it behaves (sort of) like ReMap.MapString.
// Build the strictly-paired chunk indexes and populate them.
// We are iterating over *match sets*; matchIndices here should be analgous
// to matchIndices in ReMap.MapString.
for _, matchIndices = range allMatchIndices {
if matchIndices == nil {
// I *think* the exception with the *All* variant here
// is the *entire* return (allMatchIndices) is nil if there
// aren't any matches; I can't imagine there'd be any feasible
// way it'd insert a nil *element* for an index mapping group.
// So just continuing here should be fine;
// this continue SHOULD be unreachable.
continue
}
// Reslice *here*, on the particular match index group.
// Grap the matchStr first; it's not currently *used* by anything but may in the future.
matchStr, ok = strIdxSlicer(
s,
*(*[2]int)(matchIndices[0:2]),
)
if len(matchIndices) == 0 || len(matchIndices) == 1 {
// No *sub*matches (capture groups) in this match, but it still matched the pattern.
if inclNoMatch {
for _, grpNm = range names {
if grpNm == "" {
continue
}
// We don't immediately return, though; we just stage out group names just in case.
// That's why we use tmpMap and not matches.
if _, ok = tmpMap[grpNm]; !ok {
tmpMap[grpNm] = nil
}
}
}
continue
}
matchIndices = matchIndices[2:]
// Reset from previous loop
endIdx = 0
grpIdx = 0
for startIdx = 0; endIdx < len(matchIndices); startIdx += 2 {
endIdx = startIdx + 2
if endIdx > len(matchIndices) {
endIdx = len(matchIndices)
}
if grpIdx >= len(names) {
break
}
si = stringIndexer{
group: grpIdx,
start: matchIndices[startIdx],
end: matchIndices[endIdx-1],
matched: true,
nm: names[grpIdx],
grpS: "",
ptrn: r.Regexp,
}
grpIdx++
// We do not include the entire match string here;
// we don't need it for this. Waste of memory.
_ = matchStr
/*
si.s = new(string)
*si.s = matchStr
*/
if si.nm == "" {
// unnamed capture group
continue
}
// sets si.matched and si.grpS
si.idxSlice(&s)
if !si.matched {
if !inclNoMatch {
continue
}
if _, ok = tmpMap[si.nm]; !ok {
if !inclNoMatchStrict {
tmpMap[si.nm] = nil
} else {
tmpMap[si.nm] = []string{""}
}
} else {
if inclNoMatchStrict {
tmpMap[si.nm] = append(tmpMap[si.nm], "")
}
}
continue
}
if _, ok = tmpMap[si.nm]; !ok {
tmpMap[si.nm] = make([]string, 0)
}
tmpMap[si.nm] = append(tmpMap[si.nm], si.grpS)
}
}
if inclNoMatch {
for _, grpNm = range names {
if grpNm == "" {
continue
}
if _, ok = tmpMap[grpNm]; !ok { if _, ok = tmpMap[grpNm]; !ok {
tmpMap[grpNm] = nil tmpMap[grpNm] = nil
} }

344
remap/funcs_remap_test.go Normal file
View File

@@ -0,0 +1,344 @@
package remap
import (
"fmt"
"reflect"
"regexp"
"testing"
)
type (
testMatcher struct {
Nm string
S string
M *ReMap
All bool
Expected map[string][][]byte
ExpectedStr map[string][]string
ParamInclNoMatch bool
ParamInclNoMatchStrict bool
ParamInclMustMatch bool
}
)
func TestRemap(t *testing.T) {
var matches map[string][][]byte
for midx, m := range []testMatcher{
// 1
testMatcher{
Nm: "No matches",
S: "this is a test",
M: &ReMap{regexp.MustCompile(``)},
Expected: nil,
},
// 2
testMatcher{
Nm: "Single mid match",
S: "This contains a single match in the middle of a string",
M: &ReMap{regexp.MustCompile(`\s+(?P<g1>match)\s+`)},
Expected: map[string][][]byte{
"g1": [][]byte{[]byte("match")},
},
},
// 3
testMatcher{
Nm: "multi mid match",
S: "This contains a single match and another match in the middle of a string",
M: &ReMap{regexp.MustCompile(`\s+(?P<g1>match) and another (?P<g1>match)\s+`)},
Expected: map[string][][]byte{
"g1": [][]byte{
[]byte("match"),
[]byte("match"),
},
},
},
// 4
testMatcher{
Nm: "line match",
S: "This\ncontains a\nsingle\nmatch\non a dedicated line",
M: &ReMap{regexp.MustCompile(`(?m)^(?P<g1>match)$`)},
Expected: map[string][][]byte{
"g1": [][]byte{
[]byte("match"),
},
},
},
// 5
testMatcher{
Nm: "multiline match",
S: "This\ncontains a\nsingle match and another\nmatch\nin the middle of a string",
M: &ReMap{regexp.MustCompile(`\s+(?P<g1>match) and another\s+(?P<g1>match)\s+`)},
All: true,
Expected: map[string][][]byte{
"g1": [][]byte{
[]byte("match"),
[]byte("match"),
},
},
},
// 6
// More closely mirrors something closer to real-life
testMatcher{
Nm: "mixed match",
S: " # No longer log hits/reqs/resps to file.\n" +
" #access_log /mnt/nginx_logs/vhost/tenant/site/access.log main;\n" +
" #error_log /mnt/nginx_logs/vhost/tenant/site/error.log;\n" +
" access_log off;\n" +
" error_log /dev/null;\n\n" +
" ssl_certificate /etc/nginx/tls/crt/tenant.pem;\n" +
" ssl_certificate_key /etc/nginx/tls/key/tenant.pem;\n\n",
M: &ReMap{regexp.MustCompile(`(?m)^\s*(?:error|access)_log\s+(?P<logpath>.+);\s*$`)},
All: true,
Expected: map[string][][]byte{
"logpath": [][]byte{
[]byte("off"),
[]byte("/dev/null"),
},
},
},
} {
if m.All {
matches = m.M.MapAll([]byte(m.S), false, false, false)
} else {
matches = m.M.Map([]byte(m.S), false, false, false)
}
t.Logf(
"#%d:\n\tsrc:\t'%s'\n\tptrn:\t'%s'\n\tmatch:\t%s\n",
midx+1,
m.S,
m.M.Regexp.String(),
testBmapToStrMap(matches),
)
if !reflect.DeepEqual(matches, m.Expected) {
t.Fatalf("Case #%d (\"%s\"): expected '%#v' != received '%#v'", midx+1, m.Nm, m.Expected, matches)
}
}
}
func TestRemapParams(t *testing.T) {
var matches map[string][][]byte
for midx, m := range []testMatcher{
testMatcher{
Nm: "",
S: "this is a test",
M: &ReMap{regexp.MustCompile(``)},
Expected: nil,
ParamInclNoMatch: false,
ParamInclNoMatchStrict: false,
ParamInclMustMatch: false,
},
testMatcher{
Nm: "",
S: "this is a test",
M: &ReMap{regexp.MustCompile(``)},
Expected: nil,
ParamInclNoMatch: false,
ParamInclNoMatchStrict: true,
ParamInclMustMatch: false,
},
testMatcher{
Nm: "",
S: "this is a test",
M: &ReMap{regexp.MustCompile(``)},
Expected: nil,
ParamInclNoMatch: false,
ParamInclNoMatchStrict: true,
ParamInclMustMatch: true,
},
testMatcher{
Nm: "",
S: "this is a test",
M: &ReMap{regexp.MustCompile(``)},
Expected: nil,
ParamInclNoMatch: false,
ParamInclNoMatchStrict: false,
ParamInclMustMatch: true,
},
testMatcher{
Nm: "",
S: "this is a test",
M: &ReMap{regexp.MustCompile(``)},
Expected: make(map[string][][]byte),
ParamInclNoMatch: true,
ParamInclNoMatchStrict: false,
ParamInclMustMatch: false,
},
testMatcher{
Nm: "",
S: "this is a test",
M: &ReMap{regexp.MustCompile(``)},
Expected: make(map[string][][]byte),
ParamInclNoMatch: true,
ParamInclNoMatchStrict: true,
ParamInclMustMatch: false,
},
testMatcher{
Nm: "",
S: "this is a test",
M: &ReMap{regexp.MustCompile(``)},
Expected: make(map[string][][]byte),
ParamInclNoMatch: true,
ParamInclNoMatchStrict: true,
ParamInclMustMatch: true,
},
testMatcher{
Nm: "",
S: "this is a test",
M: &ReMap{regexp.MustCompile(``)},
Expected: make(map[string][][]byte),
ParamInclNoMatch: true,
ParamInclNoMatchStrict: false,
ParamInclMustMatch: true,
},
} {
if m.All {
matches = m.M.MapAll([]byte(m.S), m.ParamInclNoMatch, m.ParamInclNoMatchStrict, m.ParamInclMustMatch)
} else {
matches = m.M.Map([]byte(m.S), m.ParamInclNoMatch, m.ParamInclNoMatchStrict, m.ParamInclMustMatch)
}
t.Logf(
"%d: %v/%v/%v: %#v\n",
midx+1, m.ParamInclNoMatch, m.ParamInclNoMatchStrict, m.ParamInclMustMatch, matches,
)
if !reflect.DeepEqual(matches, m.Expected) {
t.Fatalf("Case #%d (\"%s\"): '%#v' != '%#v'", midx+1, m.Nm, m.ExpectedStr, matches)
}
}
}
func TestRemapString(t *testing.T) {
var matches map[string][]string
for midx, m := range []testMatcher{
// 1
testMatcher{
Nm: "No matches",
S: "this is a test",
M: &ReMap{regexp.MustCompile(``)},
ExpectedStr: nil,
},
// 2
testMatcher{
Nm: "Single mid match",
S: "This contains a single match in the middle of a string",
M: &ReMap{regexp.MustCompile(`\s+(?P<g1>match)\s+`)},
ExpectedStr: map[string][]string{
"g1": []string{"match"},
},
},
// 3
testMatcher{
Nm: "multi mid match",
S: "This contains a single match and another match in the middle of a string",
M: &ReMap{regexp.MustCompile(`\s+(?P<g1>match) and another (?P<g1>match)\s+`)},
ExpectedStr: map[string][]string{
"g1": []string{
"match",
"match",
},
},
},
// 4
testMatcher{
Nm: "line match",
S: "This\ncontains a\nsingle\nmatch\non a dedicated line",
M: &ReMap{regexp.MustCompile(`(?m)^(?P<g1>match)$`)},
ExpectedStr: map[string][]string{
"g1": []string{
"match",
},
},
},
// 5
testMatcher{
Nm: "multiline match",
S: "This\ncontains a\nsingle match and another\nmatch\nin the middle of a string",
M: &ReMap{regexp.MustCompile(`\s+(?P<g1>match) and another\s+(?P<g1>match)\s+`)},
All: true,
ExpectedStr: map[string][]string{
"g1": []string{
"match",
"match",
},
},
},
// 6
// More closely mirrors something closer to real-life
testMatcher{
Nm: "mixed match",
S: " # No longer log hits/reqs/resps to file.\n" +
" #access_log /mnt/nginx_logs/vhost/tenant/site/access.log main;\n" +
" #error_log /mnt/nginx_logs/vhost/tenant/site/error.log;\n" +
" access_log off;\n" +
" error_log /dev/null;\n\n" +
" ssl_certificate /etc/nginx/tls/crt/tenant.pem;\n" +
" ssl_certificate_key /etc/nginx/tls/key/tenant.pem;\n\n",
M: &ReMap{regexp.MustCompile(`(?m)^\s*(?:error|access)_log\s+(?P<logpath>.+);\s*$`)},
All: true,
ExpectedStr: map[string][]string{
"logpath": []string{
"off",
"/dev/null",
},
},
},
} {
if m.All {
matches = m.M.MapStringAll(m.S, false, false, false)
} else {
matches = m.M.MapString(m.S, false, false, false)
}
t.Logf(
"#%d:\n\tsrc:\t'%s'\n\tptrn:\t'%s'\n\tmatch:\t%s\n",
midx+1,
m.S,
m.M.Regexp.String(),
testSmapToStrMap(matches),
)
if !reflect.DeepEqual(matches, m.ExpectedStr) {
t.Fatalf("Case #%d (\"%s\"): '%#v' != '%#v'", midx+1, m.Nm, m.ExpectedStr, matches)
}
}
}
func testBmapToStrMap(bmap map[string][][]byte) (s string) {
if bmap == nil {
return
}
s = "\n"
for k, v := range bmap {
s += fmt.Sprintf("\t%s\n", k)
for _, i := range v {
s += fmt.Sprintf("\t\t%s\n", string(i))
}
}
return
}
func testSmapToStrMap(smap map[string][]string) (s string) {
if smap == nil {
return
}
s = "\n"
for k, v := range smap {
s += fmt.Sprintf("\t%s\n", k)
for _, i := range v {
s += fmt.Sprintf("\t\t%s\n", i)
}
}
return
}

View File

@@ -0,0 +1,34 @@
package remap
// idx returns []int{s.start, s.end}.
func (s *stringIndexer) idx() (i []int) {
return []int{s.start, s.end}
}
// idxStrict returns [2]int{s.start, s.end}.
func (s *stringIndexer) idxStrict() (i [2]int) {
return [2]int{s.start, s.end}
}
/*
idxSlice populates s.grpS using s.start and s.end.
If str is nil, it will use s.s.
If str is nil and s.s is nil, it will panic with [ErrNoStr].
If the pattern does not match (s.start < 0 or s.end < 0),
s.matched will be set to false (otherwise true).
*/
func (s *stringIndexer) idxSlice(str *string) {
if str == nil {
if s.s == nil {
panic(ErrNoStr)
}
str = s.s
}
s.grpS, s.matched = strIdxSlicer(*str, s.idxStrict())
return
}

View File

@@ -5,7 +5,7 @@ import (
) )
type ( type (
// ReMap provides some map-related functions around a regexp.Regexp. // ReMap provides some map-related functions around a [regexp.Regexp].
ReMap struct { ReMap struct {
*regexp.Regexp *regexp.Regexp
} }
@@ -24,4 +24,45 @@ type (
} }
*/ */
stringIndexer struct {
// group is the capture group index for this match.
group int
// start is the string index (from the original string) where the matched group starts
start int
// end is the string index where the matched group ends
end int
/*
matched indicates if explicitly no match was found.
(This is normally indeterminate with string regex returns,
as e.g. `(?P<mygrp>\s*)`, `(?P<mygrp>(?:somestring)?)`, etc. all can be a *matched* "".)
If grpS == "" and matched == true, it DID match an empty string.
If grpS == "" and matched == false, it DID NOT MATCH the pattern.
If grpS != "", matched can be completely disregarded.
*/
matched bool
// nm is the match group name.
nm string
/*
grpS is the actual group-matched *substring*.
It will ALWAYS be either:
* the entirety of s
* a substring of s
* an empty string
it will never, and cannot be, a SUPERset of s.
it may not always be included/populated to save on memory.
*/
grpS string
/*
s is the *entire* MATCHED (sub)string.
It may not always be populated if not needed to save memory.
*/
s *string
// ptrn is the pattern applied to s.
ptrn *regexp.Regexp
}
) )

View File

@@ -4,8 +4,3 @@ const (
// DefMaskStr is the string used as the default maskStr if left empty in [Redact]. // DefMaskStr is the string used as the default maskStr if left empty in [Redact].
DefMaskStr string = "***" DefMaskStr string = "***"
) )
const (
// DefIndentStr is the string used as the default indent if left empty in [Indent].
DefIndentStr string = "\t"
)

View File

@@ -1,4 +1,17 @@
/* /*
Package stringsx aims to extend functionality of the stdlib [strings] module. Package stringsx aims to extend functionality of the stdlib [strings] module.
Note that if you need a way of mimicking Bash's shell quoting rules, [desertbit/shlex] or [buildkite/shellwords]
would be better options than [google/shlex] but this package does not attempt to reproduce
any of that functionality.
For line splitting, one should use [muesli/reflow/wordwrap].
Likewise for indentation, one should use [muesli/reflow/indent].
[desertbit/shlex]: https://pkg.go.dev/github.com/desertbit/go-shlex
[buildkite/shellwords]: https://pkg.go.dev/github.com/buildkite/shellwords
[google/shlex]: https://pkg.go.dev/github.com/google/shlex
[muesli/reflow/wordwrap]: https://pkg.go.dev/github.com/muesli/reflow/wordwrap
[muesli/reflow/indent]: https://pkg.go.dev/github.com/muesli/reflow/indent
*/ */
package stringsx package stringsx

View File

@@ -1,96 +1,170 @@
package stringsx package stringsx
import ( import (
`fmt`
`strings` `strings`
`unicode` `unicode`
) )
/* /*
Indent takes string s and indents it with string `indent` `level` times. LenSplit formats string `s` to break at, at most, every `width` characters.
If indent is an empty string, [DefIndentStr] will be used. Any existing newlines (e.g. \r\n) will be removed during a string/
substring/line's length calculation. (e.g. `foobarbaz\n` and `foobarbaz\r\n` are
both considered to be lines of length 9, not 10 and 11 respectively).
If ws is true, lines consisting of only whitespace will be indented as well. This also means that any newlines (\n or \r\n) are inherently removed from
(To then trim any extraneous trailing space, you may want to use [TrimSpaceRight] `out` (even if included in `wordWrap`; see below).
or [TrimLines].)
If empty is true, lines with no content will be replaced with lines that purely Note that if `s` is multiline (already contains newlines), they will be respected
consist of (indent * level) (otherwise they will be left as empty lines). as-is - that is, if a line ends with less than `width` chars and then has a newline,
it will be preserved as an empty element. That is to say:
This function can also be used to prefix lines with arbitrary strings as well. "foo\nbar\n\n" → []string{"foo", "bar", ""}
e.g: "foo\n\nbar\n" → []string{"foo", "", "bar"}
Indent("foo\nbar\nbaz\n", "# ", 1, false, false) This splitter is particularly simple. If you need wordwrapping, it should be done
with e.g. [github.com/muesli/reflow/wordwrap].
would yield:
# foo
# bar
# baz
<empty line>
thus allowing you to "comment out" multiple lines at once.
*/ */
func Indent(s, indent string, level uint, ws, empty bool) (indented string) { func LenSplit(s string, width uint) (out []string) {
var i string var end int
var nl string var line string
var endsNewline bool var lineRunes []rune
var sb strings.Builder
var lineStripped string
if indent == "" { if width == 0 {
indent = DefIndentStr out = []string{s}
}
// This condition functionally won't do anything, so just return the input as-is.
if level == 0 {
indented = s
return return
} }
i = strings.Repeat(indent, int(level)) for line = range strings.Lines(s) {
line = strings.TrimRight(line, "\n")
line = strings.TrimRight(line, "\r")
// This condition functionally won't do anything, so just return the input as-is. lineRunes = []rune(line)
if s == "" {
if empty {
indented = i
}
return
}
for line := range strings.Lines(s) { if uint(len(lineRunes)) <= width {
lineStripped = strings.TrimSpace(line) out = append(out, line)
nl = getNewLine(line)
endsNewline = nl != ""
// fmt.Printf("%#v => %#v\n", line, lineStripped)
if lineStripped == "" {
// fmt.Printf("WS/EMPTY LINE (%#v) (ws %v, empty %v): \n", s, ws, empty)
if line != (lineStripped + nl) {
// whitespace-only line
if ws {
sb.WriteString(i)
}
} else {
// empty line
if empty {
sb.WriteString(i)
}
}
sb.WriteString(line)
continue continue
} }
// non-empty/non-whitespace-only line.
sb.WriteString(i + line) for i := 0; i < len(lineRunes); i += int(width) {
end = i + int(width)
if end > len(lineRunes) {
end = len(lineRunes)
}
out = append(out, string(lineRunes[i:end]))
}
} }
// If it ends with a trailing newline and nothing after, strings.Lines() will skip the last (empty) line. return
if endsNewline && empty { }
nl = getNewLine(s)
sb.WriteString(i) /*
LenSplitStr wraps [LenSplit] but recombines into a new string with newlines.
It's mostly just a convenience wrapper.
All arguments remain the same as in [LenSplit] with an additional one,
`winNewLine`, which if true will use \r\n as the newline instead of \n.
*/
func LenSplitStr(s string, width uint, winNewline bool) (out string) {
var outSl []string = LenSplit(s, width)
if winNewline {
out = strings.Join(outSl, "\r\n")
} else {
out = strings.Join(outSl, "\n")
} }
indented = sb.String() return
}
/*
Pad pads each element in `s` to length `width` using `pad`.
If `pad` is empty, a single space (0x20) will be assumed.
Note that `width` operates on rune size, not byte size.
(In ASCII, they will be the same size.)
If a line in `s` is greater than or equal to `width`,
no padding will be performed.
If `leftPad` is true, padding will be applied to the "left" (beginning")
of each element instead of the "right" ("end").
*/
func Pad(s []string, width uint, pad string, leftPad bool) (out []string) {
var idx int
var padIdx int
var runeIdx int
var padLen uint
var elem string
var unpadLen uint
var tmpPadLen int
var padRunes []rune
var tmpPad []rune
if width == 0 {
out = s
return
}
out = make([]string, len(s))
// Easy; supported directly in fmt.
if pad == "" {
for idx, elem = range s {
if leftPad {
out[idx] = fmt.Sprintf("%*s", width, elem)
} else {
out[idx] = fmt.Sprintf("%-*s", width, elem)
}
}
return
}
// This gets a little more tricky.
padRunes = []rune(pad)
padLen = uint(len(padRunes))
for idx, elem = range s {
// First we need to know the number of runes in elem.
unpadLen = uint(len([]rune(elem)))
// If it's more than/equal to width, as-is.
if unpadLen >= width {
out[idx] = elem
} else {
// Otherwise, we need to construct/calculate a pad.
if (width-unpadLen)%padLen == 0 {
// Also easy enough.
if leftPad {
out[idx] = fmt.Sprintf("%s%s", strings.Repeat(pad, int((width-unpadLen)/padLen)), elem)
} else {
out[idx] = fmt.Sprintf("%s%s", elem, strings.Repeat(pad, int((width-unpadLen)/padLen)))
}
} else {
// This is where it gets a little hairy.
tmpPad = []rune{}
tmpPadLen = int(width - unpadLen)
idx = 0
padIdx = 0
for runeIdx = range tmpPadLen {
tmpPad[runeIdx] = padRunes[padIdx]
if uint(padIdx) >= padLen {
padIdx = 0
} else {
padIdx++
}
runeIdx++
}
if leftPad {
out[idx] = fmt.Sprintf("%s%s", string(tmpPad), elem)
} else {
out[idx] = fmt.Sprintf("%s%s", elem, string(tmpPad))
}
}
}
}
return return
} }
@@ -118,6 +192,9 @@ As a safety precaution, if:
len(s) <= (leading + trailing) len(s) <= (leading + trailing)
then the entire string will be *masked* and no unmasking will be performed. then the entire string will be *masked* and no unmasking will be performed.
Note that this DOES NOT do a string *replace*, it provides a masked version of `s` itself.
Wrap Redact with [strings.ReplaceAll] if you want to replace a certain value with a masked one.
*/ */
func Redact(s, maskStr string, leading, trailing uint, newlines bool) (redacted string) { func Redact(s, maskStr string, leading, trailing uint, newlines bool) (redacted string) {
@@ -218,7 +295,7 @@ func TrimLines(s string, left, right bool) (trimmed string) {
return return
} }
// TrimSpaceLeft is like [strings.TrimSpace] but only removes leading whitespace from string s. // TrimSpaceLeft is like [strings.TrimSpace] but only removes leading whitespace from string `s`.
func TrimSpaceLeft(s string) (trimmed string) { func TrimSpaceLeft(s string) (trimmed string) {
trimmed = strings.TrimLeftFunc(s, unicode.IsSpace) trimmed = strings.TrimLeftFunc(s, unicode.IsSpace)
@@ -236,7 +313,7 @@ func TrimSpaceRight(s string) (trimmed string) {
return return
} }
// getNewLine is too unpredictable to be used outside of this package so it isn't exported. // getNewLine is too unpredictable/nuanced to be used as part of a public API promise so it isn't exported.
func getNewLine(s string) (nl string) { func getNewLine(s string) (nl string) {
if strings.HasSuffix(s, "\r\n") { if strings.HasSuffix(s, "\r\n") {

View File

@@ -37,113 +37,6 @@ type (
} }
) )
func TestIndent(t *testing.T) {
var out string
var tests []testIndentSet = []testIndentSet{
testIndentSet{
name: "standard, no trailing newline",
orig: "foo\nbar\nbaz",
indent: "",
lvl: 1,
ws: false,
empty: false,
tgt: "\tfoo\n\tbar\n\tbaz",
},
testIndentSet{
name: "standard, trailing newline",
orig: "foo\nbar\nbaz\n",
indent: "",
lvl: 1,
ws: false,
empty: false,
tgt: "\tfoo\n\tbar\n\tbaz\n",
},
testIndentSet{
name: "standard, trailing newline with empty",
orig: "foo\nbar\nbaz\n",
indent: "",
lvl: 1,
ws: false,
empty: true,
tgt: "\tfoo\n\tbar\n\tbaz\n\t",
},
testIndentSet{
name: "standard, trailing newline with ws",
orig: "foo\nbar\nbaz\n",
indent: "",
lvl: 1,
ws: true,
empty: false,
tgt: "\tfoo\n\tbar\n\tbaz\n",
},
testIndentSet{
name: "standard, trailing newline with ws and empty",
orig: "foo\nbar\nbaz\n",
indent: "",
lvl: 1,
ws: true,
empty: true,
tgt: "\tfoo\n\tbar\n\tbaz\n\t",
},
testIndentSet{
name: "standard, trailing ws newline with empty",
orig: "foo\nbar\nbaz\n ",
indent: "",
lvl: 1,
ws: false,
empty: true,
tgt: "\tfoo\n\tbar\n\tbaz\n ",
},
testIndentSet{
name: "standard, trailing ws newline with ws",
orig: "foo\nbar\nbaz\n ",
indent: "",
lvl: 1,
ws: true,
empty: false,
tgt: "\tfoo\n\tbar\n\tbaz\n\t ",
},
testIndentSet{
name: "standard, trailing ws newline with ws and empty",
orig: "foo\nbar\nbaz\n \n",
indent: "",
lvl: 1,
ws: true,
empty: true,
tgt: "\tfoo\n\tbar\n\tbaz\n\t \n\t",
},
testIndentSet{
name: "comment",
orig: "foo\nbar\nbaz",
indent: "# ",
lvl: 1,
ws: false,
empty: false,
tgt: "# foo\n# bar\n# baz",
},
}
for idx, ts := range tests {
out = Indent(ts.orig, ts.indent, ts.lvl, ts.ws, ts.empty)
if out == ts.tgt {
t.Logf("[%d] OK (%s): %#v: got %#v", idx, ts.name, ts.orig, out)
} else {
t.Errorf(
"[%d] FAIL (%s): %#v (len %d):\n"+
"\t\t\texpected (len %d): %#v\n"+
"\t\t\tgot (len %d): %#v\n"+
"\t\t%#v",
idx, ts.name, ts.orig, len(ts.orig),
len(ts.tgt), ts.tgt,
len(out), out,
ts,
)
}
}
}
func TestRedact(t *testing.T) { func TestRedact(t *testing.T) {
var out string var out string

4
timex/doc.go Normal file
View File

@@ -0,0 +1,4 @@
/*
Package timex provides some handy [time]-related functions.
*/
package timex

35
timex/funcs.go Normal file
View File

@@ -0,0 +1,35 @@
package timex
import (
`time`
)
/*
F64Seconds returns [time.Time] `t` as a 64-bit float of <seconds>.<nanoseconds>
(where <nanoseconds> is the number of nanoseconds since <seconds>,
and <seconds> is the number of seconds since the UNIX epoch).
This can be used to represent a UNIX Epoch timestamp as seconds but with nanosecond precision.
*/
func F64Seconds(t time.Time) (f64 float64) {
return F64Nanoseconds(t) / float64(time.Second)
}
/*
F64Milliseconds is like [F64Seconds] but with a millisecond integer.
*/
func F64Milliseconds(t time.Time) (f64 float64) {
return F64Nanoseconds(t) / float64(time.Millisecond)
}
/*
F64Microseconds is like [F64Seconds] but with a microsecond integer.
*/
func F64Microseconds(t time.Time) (f64 float64) {
return F64Nanoseconds(t) / float64(time.Microsecond)
}
// F64Nanoseconds returns [time.Time.UnixNano] as a float64.
func F64Nanoseconds(t time.Time) (f64 float64) {
return float64(t.UnixNano())
}

30
timex/funcs_test.go Normal file
View File

@@ -0,0 +1,30 @@
package timex
import (
"testing"
`time`
)
func TestF64(t *testing.T) {
var tmNano float64 = 1766533329999999999
var tmSeconds float64 = 1766533329.999999999
var tmMilli float64 = 1766533329999.999999
var tmMicro float64 = 1766533329999999.999
// 2025-12-23 23:42:09.999999999 +0000 UTC
var tm time.Time = time.Unix(1766533329, int64(time.Second-1))
if F64Seconds(tm) != tmSeconds {
t.Fatalf("Failed seconds: %f != %f", F64Seconds(tm), tmSeconds)
}
if F64Milliseconds(tm) != tmMilli {
t.Fatalf("Failed milliseconds: %f != %f", F64Milliseconds(tm), tmMilli)
}
if F64Microseconds(tm) != tmMicro {
t.Fatalf("Failed microseconds: %f != %f", F64Microseconds(tm), tmMicro)
}
if F64Nanoseconds(tm) != tmNano {
t.Fatalf("Failed nanoseconds: %f != %f", F64Nanoseconds(tm), tmNano)
}
}

6
tplx/consts.go Normal file
View File

@@ -0,0 +1,6 @@
package tplx
const (
TplTypeText tplType = iota
TplTypeHtml
)

4
tplx/doc.go Normal file
View File

@@ -0,0 +1,4 @@
/*
Package tplx provides some "shortcuts" to [text/template] and [html/template] rendering.
*/
package tplx

9
tplx/errs.go Normal file
View File

@@ -0,0 +1,9 @@
package tplx
import (
`errors`
)
var (
ErrInvalidTplType = errors.New("unknown/invalid template type")
)

235
tplx/funcs.go Normal file
View File

@@ -0,0 +1,235 @@
package tplx
import (
`bytes`
htmlTpl `html/template`
txtTpl `text/template`
)
// MustTplStrToStr wraps [TplStrToStr] but will panic on a non-nil error instead of returning it.
func MustTplStrToStr(tplStr string, typ tplType, obj any) (s string) {
var err error
if s, err = TplStrToStr(tplStr, typ, obj); err != nil {
panic(err)
}
return
}
// MustTplToStr wraps [TplToStr] but will panic on error instead of returning it.
func MustTplToStr[T Template](tpl T, obj any) (s string) {
var err error
if s, err = TplToStr(tpl, obj); err != nil {
panic(err)
}
return
}
// MustTplToStrWith wraps [TplToStrWith] but will panic on error instead of returning it.
func MustTplToStrWith[T Template](tpl T, tplNm string, obj any) (s string) {
var err error
if s, err = TplToStrWith(tpl, tplNm, obj); err != nil {
panic(err)
}
return
}
/*
TplStrToStr takes in a template string, a template type (see i.e. [TplTypeText], [TplTypeHtml]),
and an object and renders to a string.
This is obviously quite inflexible - there's no way to provide a [text/template.FuncMap]/[html/template.FuncMap],
for instance, but if more advanced template features aren't needed then this might just do the trick.
If you need something more flexible, see [TplToStr] instead.
*/
func TplStrToStr(tplStr string, typ tplType, obj any) (out string, err error) {
var ttpl *txtTpl.Template
var htpl *htmlTpl.Template
var buf *bytes.Buffer = new(bytes.Buffer)
switch typ {
case TplTypeText:
if ttpl, err = txtTpl.New("").Parse(tplStr); err != nil {
return
}
if err = ttpl.Execute(buf, obj); err != nil {
return
}
case TplTypeHtml:
if htpl, err = htmlTpl.New("").Parse(tplStr); err != nil {
return
}
if err = htpl.Execute(buf, obj); err != nil {
return
}
default:
err = ErrInvalidTplType
return
}
out = buf.String()
return
}
/*
TplToStr takes in an [html/template] or [text/template] and an object and executes it.
PLEASE NOTE that it is expected that `tpl` has already had at least one template string `.Parse()`'d in.
If you haven't used generics in Golang yet, this function would be used via something like the following complete example
for both a [text/template.Template] (import-aliased as `txtT.Template`) and
an [html/template.Template] (import-aliased as `htmlT.Template`).
import (
"fmt"
"log"
txtT "text/template"
htmlT "html/template"
`r00t2.io/goutils/tplx`
)
type (
S struct {
Name string
}
)
var (
tTpl *txtT.Template
hTpl *htmlT.Template
)
const tTplStr string = "Greetings, {{ .Name }}!\n"
const hTplStr string = `<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<title>Hello, {{ .Name }}!</title>
</head>
<body>
<p>Hello, {{ .Name }}. Good to see you.</p>
</body>
</html>
`
func main() {
var err error
var s string
var o *S
o = &S{
Name: "Bob",
}
// A text template.
if tTpl, err = txtT.
New("my_txt_template").
Parse(tTplStr); err != nil {
log.Panicf("Failed to parse text template string '%s': %v\n", tTplStr, err)
}
if s, err = tplx.TplToStr[*txtT.Template](tTpl, o); err != nil {
log.Panicf("Failed to render text template to string: %v\n", err)
}
fmt.Println(s)
// An HTML template.
if hTpl, err = htmlT.
New("index.html").
Parse(hTplStr); err != nil {
log.Panicf("Failed to parse HTML template string '%s': %v\n", hTplStr, err)
}
if s, err = tplx.TplToStr[*htmlT.Template](hTpl, o); err != nil {
log.Panicf("Failed to render HTML template to string: %v\n", err)
}
fmt.Println(s)
}
Additionally, because this function uses a union type [Template],
you can even leave the type indicator off.
For example:
// ...
if s, err = tplx.TplToStr(tTpl, o); err != nil {
log.Panicf("Failed to render text template to string: %v\n", err)
}
// ...
if s, err = tplx.TplToStr(hTpl, o); err != nil {
log.Panicf("Failed to render HTML template to string: %v\n", err)
}
// ...
However, this is not recommended for readability purposes - including
the type indicator indicates (heh heh) to others reading your code
what type `tTpl` and `hTpl` are without needing to cross-reference
their declaration/assignment/definition.
For more information on generics in Golang, see:
* The introductory [blog post]
* The official [tutorial]
* The syntax [reference doc]
* The (community-maintained/unofficial) [Go by Example: Generics]
[blog post]: https://go.dev/blog/intro-generics
[tutorial]: https://go.dev/doc/tutorial/generics
[reference doc]: https://go.dev/ref/spec#Instantiations
[Go by Example: Generics]: https://gobyexample.com/generics
*/
func TplToStr[T Template](tpl T, obj any) (out string, err error) {
var buf *bytes.Buffer = new(bytes.Buffer)
if err = tpl.Execute(buf, obj); err != nil {
return
}
out = buf.String()
return
}
/*
TplToStrWith functions the exact same as [TplToStr] but allows you to specify the
template entry point (template name) named `nm`.
For example (see [TplToStr] for a full example):
// ...
var tplNm string = "index.html"
if s, err = tplx.TplToStrWith(tTpl, tplNm, o); err != nil {
log.Panicf("Failed to render HTML template '%s' to string: %v\n", tplNm, err)
}
// ...
would call the equivalent of:
// ...
if err = tpl.ExecuteTemplate(<internal buffer>, tplNm, o); err != nil {
// ...
}
*/
func TplToStrWith[T Template](tpl T, tplNm string, obj any) (out string, err error) {
var buf *bytes.Buffer = new(bytes.Buffer)
if err = tpl.ExecuteTemplate(buf, tplNm, obj); err != nil {
return
}
out = buf.String()
return
}

103
tplx/funcs_test.go Normal file
View File

@@ -0,0 +1,103 @@
package tplx
import (
htmlT `html/template`
`log`
"testing"
txtT `text/template`
)
const (
txtTplNm string = "my_txt_template"
htmlTplNm string = "index.html"
tgtTxt string = "Greetings, Bob!\n"
tgtHtml string = "<!DOCTYPE html>\n<html lang=\"en\">\n\t<head>\n\t\t<meta charset=\"utf-8\">\n\t\t<title>Hello, Bob!</title>\n\t</head>\n\t<body>\n\t\t<p>Hello, Bob. Good to see you.</p>\n\t</body>\n</html>\n"
tTplStr string = "Greetings, {{ .Name }}!\n"
hTplStr string = `<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<title>Hello, {{ .Name }}!</title>
</head>
<body>
<p>Hello, {{ .Name }}. Good to see you.</p>
</body>
</html>
`
)
var (
tTpl *txtT.Template = txtT.Must(txtT.New(txtTplNm).Parse(tTplStr))
hTpl *htmlT.Template = htmlT.Must(htmlT.New(htmlTplNm).Parse(hTplStr))
o struct{ Name string } = struct{ Name string }{
Name: "Bob",
}
)
func TestTpl(t *testing.T) {
var err error
var s string
// if s, err = TplToStr[*txtT.Template](tTpl, o); err != nil {
if s, err = TplToStr(tTpl, o); err != nil {
t.Fatalf("Failed to render text template to string: %v\n", err)
}
t.Logf("Text template (%#v): '%s'", s, s)
if s != tgtTxt {
t.Fatalf("Mismatch on text template '%s'", s)
}
// if s, err = TplToStr[*htmlT.Template](hTpl, o); err != nil {
if s, err = TplToStr(hTpl, o); err != nil {
log.Panicf("Failed to render HTML template to string: %v\n", err)
}
t.Logf("HTML template (%#v):\n%s", s, s)
if s != tgtHtml {
t.Fatalf("Mismatch on HTML template '%s'", s)
}
}
func TestTplStr(t *testing.T) {
var err error
var s string
if s, err = TplStrToStr(tTplStr, TplTypeText, o); err != nil {
t.Fatalf("Failed to render text template to string: %v\n", err)
}
t.Logf("Text template (%#v): '%s'", s, s)
if s != tgtTxt {
t.Fatalf("Mismatch on text template '%s'", s)
}
if s, err = TplStrToStr(hTplStr, TplTypeHtml, o); err != nil {
log.Panicf("Failed to render HTML template to string: %v\n", err)
}
t.Logf("HTML template (%#v):\n%s", s, s)
if s != tgtHtml {
t.Fatalf("Mismatch on HTML template '%s'", s)
}
}
func TestTplWith(t *testing.T) {
var err error
var s string
if s, err = TplToStrWith(tTpl, txtTplNm, o); err != nil {
t.Fatalf("Failed to render text template to string: %v\n", err)
}
t.Logf("Text template (%#v): '%s'", s, s)
if s != tgtTxt {
t.Fatalf("Mismatch on text template '%s'", s)
}
if s, err = TplToStrWith(hTpl, htmlTplNm, o); err != nil {
log.Panicf("Failed to render HTML template to string: %v\n", err)
}
t.Logf("HTML template (%#v):\n%s", s, s)
if s != tgtHtml {
t.Fatalf("Mismatch on HTML template '%s'", s)
}
}

19
tplx/types.go Normal file
View File

@@ -0,0 +1,19 @@
package tplx
import (
htmlTpl `html/template`
`io`
txtTpl `text/template`
)
type (
tplType uint8
)
type (
Template interface {
*txtTpl.Template | *htmlTpl.Template
Execute(w io.Writer, obj any) (err error)
ExecuteTemplate(w io.Writer, tplNm string, obj any) (err error)
}
)