Compare commits

..

1 Commits

Author SHA1 Message Date
brent saner c9f3e7a639 v1.16.11
ADDED:
* stringsx:
** RunesInString
** RuneMapFromString
** SquashConsec
** SquashConsecRunes
** SquashConsecRunesAll
** SquashMap
** SquashWhitespace
2026-07-13 19:17:39 -04:00
8 changed files with 1006 additions and 2 deletions
+94
View File
@@ -0,0 +1,94 @@
package main
import (
"embed"
"os"
"slices"
"github.com/deckarep/golang-set/v2"
"r00t2.io/sysutils/paths"
)
// NewEmbedFsMember returns an embedFsMember from embedFs.
func NewEmbedFsMember(embedFs *embed.FS) (m *embedFsMember) {
m = &embedFsMember{
FS: embedFs,
}
return
}
/*
NewMergedFS returns a MergedFS.
The last filesystem in a MergedFS is always embedTplFS.
If fsPaths is empty, mfs will only contain embedTplFS.
Nonexisting paths in fsPaths will be skipped (and added to the noExist slice).
Order will be retained but duplicates will be skipped and added to the dupes slice.
noExist and dupes will be nil if no non-existing or duplicate paths are encountereed.
*/
func NewMergedFS(fsPaths ...string) (mfs *MergedFS, noExist, dupes []string, err error) {
var m MergedFS
var exists bool
var fsPath string
var fsMem mergedFsMember
m = MergedFS{
fses: make([]mergedFsMember, 0, len(fsPaths)+1),
overridePaths: make([]string, 0, len(fsPaths)+1),
uniq: mapset.NewSet[string](),
}
for _, fsPath = range fsPaths {
if exists, err = paths.RealPathExists(&fsPath); err != nil {
return
}
if !exists {
noExist = append(noExist, fsPath)
continue
}
if !m.uniq.Add(fsPath) {
dupes = append(dupes, fsPath)
continue
}
if fsMem, err = NewRealFsMember(fsPath); err != nil {
return
}
m.fses = append(m.fses, fsMem)
m.overridePaths = append(m.overridePaths, fsPath)
}
// embedded fs last
m.fses = append(m.fses, NewEmbedFsMember(&embedTplFS))
m.overridePaths = append(m.overridePaths, "")
m.fses = slices.Clip(m.fses)
m.overridePaths = slices.Clip(m.overridePaths)
mfs = &m
return
}
// NewRealFsMember returns a realFsMember (which properly conforms to mergedFsMember) from a real filesystem path.
func NewRealFsMember(fsPath string) (fsm *realFsMember, err error) {
var fsp string
var m realFsMember
if err = paths.RealPath(&fsp); err != nil {
return
}
if m.Root, err = os.OpenRoot(fsPath); err != nil {
return
}
fsm = &m
return
}
@@ -0,0 +1,18 @@
package main
import (
"io/fs"
)
// Close is a no-op, but used to conform to mergedFsMember.
func (e *embedFsMember) Close() (err error) { return }
// Stat wraps fs.Stat on the embeded filesystem.
func (e *embedFsMember) Stat(fnm string) (stat fs.FileInfo, err error) {
if stat, err = fs.Stat(e.FS, fnm); err != nil {
return
}
return
}
@@ -0,0 +1,360 @@
package main
import (
"io"
"io/fs"
"os"
"path/filepath"
"slices"
"r00t2.io/sysutils/paths"
)
/*
Append adds fsPaths as fs.FS to the MergedFS (before the embed.FS).
If fsPaths is empty, Append is a NO-OP.
noExist and dupes have the same meaning as in NewMergedFS.
*/
func (m *MergedFS) Append(fsPaths ...string) (noExist, dupes []string, err error) {
var exists bool
var fsPath string
var newPaths []string
var fsMem *realFsMember
var newFs []mergedFsMember
m.mut.Lock()
defer m.mut.Unlock()
newPaths = make([]string, 0, len(fsPaths))
newFs = make([]mergedFsMember, 0, len(fsPaths))
for _, fsPath = range fsPaths {
if exists, err = paths.RealPathExists(&fsPath); err != nil {
return
}
if !exists {
noExist = append(noExist, fsPath)
continue
}
if fsMem, err = NewRealFsMember(fsPath); err != nil {
return
}
if !m.uniq.Add(fsPath) {
dupes = append(dupes, fsPath)
continue
}
newFs = append(newFs, fsMem)
newPaths = append(newPaths, fsPath)
}
if len(newFs) == 0 {
return
}
m.overridePaths = append(m.overridePaths, newPaths...)
m.fses = append(m.fses[:len(m.fses)-1], append(newFs, m.fses[len(m.fses)-1])...)
return
}
// Close closes all realFsMember members. This MergedFS will not be able to be used after this is called.
func (m *MergedFS) Close() (err error) {
var fsMem mergedFsMember
m.mut.Lock()
defer m.mut.Unlock()
for _, fsMem = range m.fses {
if err = fsMem.(io.Closer).Close(); err != nil {
return
}
}
return
}
/*
Insert adds fsPaths as fs.FS to a MergedFS starting at index idx.
If fsPaths is empty, Insert is a NO-OP.
If idx is 0 or negative, they will be prepended.
If idx is larger than the last index of real directories (excluding the last embed.FS),
it will be reduced to the highest valid index.
noExist and dupes have the same meaning as in NewMergedFS.
*/
func (m *MergedFS) Insert(idx int, fsPaths ...string) (noExist, dupes []string, err error) {
var exists bool
var fsPath string
var newPaths []string
var fsMem *realFsMember
var newFs []mergedFsMember
if len(fsPaths) == 0 {
return
}
m.mut.Lock()
defer m.mut.Unlock()
newPaths = make([]string, 0, len(fsPaths))
newFs = make([]mergedFsMember, 0, len(fsPaths))
for _, fsPath = range fsPaths {
if exists, err = paths.RealPathExists(&fsPath); err != nil {
return
}
if !exists {
noExist = append(noExist, fsPath)
continue
}
if fsMem, err = NewRealFsMember(fsPath); err != nil {
return
}
if !m.uniq.Add(fsPath) {
dupes = append(dupes, fsPath)
continue
}
newPaths = append(newPaths, fsPath)
newFs = append(newFs, fsMem)
}
if len(newFs) == 0 {
return
}
if idx < 0 {
m.fses = append(newFs, m.fses...)
m.overridePaths = append(newPaths, m.overridePaths...)
return
} else if idx >= len(m.fses)-1 {
idx = len(m.fses) - 2
}
m.fses = append(m.fses[:idx], append(newFs, m.fses[idx:]...)...)
m.overridePaths = append(m.overridePaths[:idx], append(newPaths, m.overridePaths[idx:]...)...)
return
}
// Open opens filename fnm and returns it as file handler fh.
func (m *MergedFS) Open(fnm string) (fh fs.File, err error) {
var fsref fs.FS
m.mut.RLock()
defer m.mut.RUnlock()
// Iterate until a match is found.
for _, fsref = range m.fses {
if fh, err = fsref.Open(fnm); err == nil {
return
}
}
// No match found.
err = &fs.PathError{
Op: "Open",
Path: fnm,
Err: fs.ErrNotExist,
}
return
}
/*
Paths returns a slice of real filesystem paths that are used by this MergedFS.
If sorted is true, they will be returned as a sorted slice -
otherwise they will be returned in the order they are checked.
*/
func (m *MergedFS) Paths(sorted bool) (fsPaths []string) {
m.mut.RLock()
defer m.mut.RUnlock()
if sorted {
fsPaths = m.uniq.ToSlice()
slices.Sort(fsPaths)
} else {
fsPaths = make([]string, len(m.overridePaths)-1)
copy(fsPaths, m.overridePaths[:len(m.overridePaths)-1])
}
return
}
/*
ReadDir returns a listing of path matcher fpath.
The result is a combination of all filesystems; the first encountered matching
name is added.
*/
func (m *MergedFS) ReadDir(fpath string) (entries []fs.DirEntry, err error) {
var idx int
var e fs.DirEntry
var tmpEntries []fs.DirEntry
var seen map[string]bool = make(map[string]bool)
m.mut.RLock()
defer m.mut.RUnlock()
// merge real and embedded, preferring first-seen real paths
for idx, _ = range m.fses {
if m.overridePaths[idx] == "" {
if tmpEntries, err = m.fses[idx].ReadDir(filepath.FromSlash(fpath)); err != nil {
continue
}
} else {
if tmpEntries, err = os.ReadDir(filepath.Join(m.overridePaths[idx], filepath.FromSlash(fpath))); err != nil {
continue
}
}
for _, e = range tmpEntries {
if seen[e.Name()] {
continue
}
seen[e.Name()] = true
entries = append(entries, e)
}
}
return
}
/*
ReadFile returns the bytes for the first found fpath.
It wraps MergedFS.Open.
*/
func (m *MergedFS) ReadFile(fpath string) (b []byte, err error) {
var fh fs.File
if fh, err = m.Open(fpath); nil != err {
return
}
defer func() {
if fh != nil {
if err = fh.Close(); err != nil {
return
}
}
}()
if b, err = io.ReadAll(fh); err != nil {
return
}
return
}
/*
Stat returns the fs.FileInfo for the first found fpath.
It wraps MergedFS.Open.
*/
func (m *MergedFS) Stat(fpath string) (stat fs.FileInfo, err error) {
var fh fs.File
if fh, err = m.Open(fpath); nil != err {
return
}
defer func() {
if fh != nil {
if err = fh.Close(); err != nil {
return
}
}
}()
if stat, err = fh.Stat(); err != nil {
return
}
return
}
/*
RemoveFsByIdx removes the filesystem found at index idx.
An attempt to remove the final embed.FS filesystem will result in an ErrFsIdxOOB.
If idx is larger than the last index of "real" filesystems, an ErrFsIdxOOB will be returned.
*/
func (m *MergedFS) RemoveFsByIdx(idx uint) (err error) {
var fsPath string
m.mut.Lock()
defer m.mut.Unlock()
if idx+1 == uint(len(m.fses)) {
return
}
if idx > uint(len(m.overridePaths)-1) {
err = ErrFsIdxOOB
return
}
fsPath = m.overridePaths[idx]
if m.uniq.Contains(fsPath) {
m.uniq.Remove(fsPath)
}
m.fses = append(m.fses[:idx], m.fses[idx+1:]...)
m.overridePaths = append(m.overridePaths[:idx], m.overridePaths[idx+1:]...)
return
}
/*
RemoveFsByPath removes the fileststem with path dirPath.
If dirpath is empty, this function NOOPs.
If dirpath does not exist in this MergedFS, an ErrFsPathNoExists will be returned.
*/
func (m *MergedFS) RemoveFsByPath(dirPath string) (err error) {
var idx int
var fsp string
var p string = dirPath
if p == "" {
return
}
if err = paths.RealPath(&p); err != nil {
return
}
m.mut.Lock()
defer m.mut.Unlock()
if !m.uniq.Contains(p) {
err = ErrFsPathNoExists
return
}
m.uniq.Remove(p)
for idx, fsp = range m.overridePaths {
if fsp == p {
m.fses = append(m.fses[:idx], m.fses[idx+1:]...)
m.overridePaths = append(m.overridePaths[:idx], m.overridePaths[idx+1:]...)
return
}
}
err = ErrFsIdxOOB
return
}
@@ -0,0 +1,52 @@
package main
import (
"io/fs"
"os"
"path/filepath"
"strings"
"r00t2.io/sysutils/paths"
)
// Open opens file fnm.
func (r *realFsMember) Open(fnm string) (fh fs.File, err error) {
if fh, err = r.Root.Open(fnm); err != nil {
return
}
return
}
// ReadDir returns directory entries at fpath joined with the realFsMember's root.
func (r *realFsMember) ReadDir(fpath string) (entries []fs.DirEntry, err error) {
var rel string
var fsp string = fpath
if !filepath.IsAbs(fsp) {
fsp = filepath.Join(
r.Root.Name(),
filepath.FromSlash(fsp),
)
}
if err = paths.RealPath(&fsp); err != nil {
return
}
if rel, err = filepath.Rel(r.Root.Name(), fsp); err != nil {
// Not relative to this root, so skip.
err = nil
return
}
if strings.HasPrefix(rel, "..") {
// Absolute but not relative to this root, so skip.
return
}
if entries, err = os.ReadDir(fsp); err != nil {
return
}
return
}
+23
View File
@@ -0,0 +1,23 @@
package main
type (
mergedFsMember interface {
fs.ReadDirFS
fs.ReadFileFS
fs.StatFS
io.Closer
}
realFsMember struct {
*os.Root
}
embedFsMember struct {
*embed.FS
}
MergedFS struct {
// always appended with embedTplFs
fses []mergedFsMember
overridePaths []string
uniq mapset.Set[string]
mut sync.RWMutex
}
)
+457
View File
@@ -8,6 +8,7 @@ import (
"slices"
"strings"
"unicode"
`unicode/utf8`
)
/*
@@ -498,6 +499,462 @@ func Reverse(s string) (revS string) {
return
}
/*
RunesInString returns an ordered slice of all unique runes in
UTF-8 string `s`.
If `s` is an empty string, runes will be a non-nil empty slice.
*/
func RunesInString(s string) (runes []rune) {
var r rune
var idx int
var runeMap map[rune]uint64 = RuneMapFromString(s)
runes = make([]rune, len(runeMap))
idx = 0
for r, _ = range runeMap {
runes[idx] = r
idx++
}
slices.Sort(runes)
return
}
/*
RuneMapFromString returns a map of unique runes found in
UTF-8 string `s` along with a count of their occurrence.
If `s` is an empty string, runes will be a non-nil empty map.
Non-UTF-8 runes are skipped.
*/
func RuneMapFromString(s string) (runes map[rune]uint64) {
var r rune
var runeSz int
runes = make(map[rune]uint64)
for idx := 0; idx < len(s); {
r, runeSz = utf8.DecodeRuneInString(s[idx:])
if r == utf8.RuneError && runeSz == 1 {
idx += runeSz
continue
}
runes[r]++
idx += runeSz
}
return
}
/*
SquashConsec squashes/collapses consecutive instances of sequences
seq to a single instance.
It is expected that s is UTF-8 or a compatible subset (e.g. ASCII).
If seq is nil/empty, any n+1 consecutive instance of *any rune*
will be squashed to a single instance. This is *much* more performant
(as it simply wraps [SquashConsecRunesAll]) at the cost of lack of scoping.
If you are trying to squash/collapse whitespace instead, [SquashWhitespace]
may be more apropos.
If you want more fine-grained control over sequence replacement, see [SquashMap].
If s is an empty string, SquashConsec will return `s` as-is.
*/
func SquashConsec(s string, seq ...string) (squashed string) {
var idx int
var sIdx int
var kLen int
var start int
var seqIdx int
var nextNum int
var nextPos int
var sb *strings.Builder
squashed = s
if len(s) == 0 {
return
}
if len(seq) == 0 {
squashed = SquashConsecRunesAll(s)
return
}
for seqIdx = range seq {
if seq[seqIdx] == "" {
continue
}
sb = new(strings.Builder)
sb.Grow(len(squashed))
kLen = len(seq[seqIdx])
for idx = 0; idx < len(squashed); {
sIdx = strings.Index(squashed[idx:], seq[seqIdx])
if sIdx < 0 {
sb.WriteString(squashed[idx:])
break
}
start = idx + sIdx
sb.WriteString(squashed[idx:start])
nextNum = 0
nextPos = start
for strings.HasPrefix(squashed[nextPos:], seq[seqIdx]) {
nextNum++
nextPos += kLen
}
if nextNum >= 1 {
sb.WriteString(seq[seqIdx])
}
idx = nextPos
}
squashed = sb.String()
}
return
}
/*
SquashConsecRunes squashes/collapses consecutive instances of sequences
seq with a single instance.
It is expected that s is UTF-8 or a compatible subset (e.g. ASCII).
If seq is nil/empty, any n+1 consecutive instance of *any rune*
will be squashed to a single instance. This is *much* more performant
(as it simply wraps [SquashConsecRunesAll]) at the cost of lack of scoping.
If s is an empty string, SquashConsecRunes will return `s` as-is.
*/
func SquashConsecRunes(s string, seq ...rune) (squashed string) {
var r rune
var idx int
var prev rune
var seqIdx int
var runeSz int
var sb *strings.Builder
squashed = s
if len(s) == 0 {
return
}
if len(seq) == 0 {
squashed = SquashConsecRunesAll(s)
return
}
for seqIdx = range seq {
if seq[seqIdx] < 0 {
continue
}
sb = new(strings.Builder)
sb.Grow(len(squashed))
prev = -1
for idx = 0; idx < len(squashed); {
r, runeSz = utf8.DecodeRuneInString(squashed[idx:])
if r == utf8.RuneError && runeSz == 1 {
prev = -1
sb.WriteString(squashed[idx : idx+runeSz])
} else {
if r != seq[seqIdx] || prev != r {
prev = r
sb.WriteRune(r)
}
}
idx += runeSz
}
squashed = sb.String()
}
return
}
/*
SquashConsecRunesAll squashes/condenses/collapses all consecutive occurrences
of every rune in `s` to a single instance.
It is expected that s is UTF-8 or a compatible subset (e.g. ASCII);
non-UTF-8 runes will be written as-is to squashed (even if duplicate).
That is to say, for an `s` of:
* `fo`
* `foo`
* `fooo`
* `foooo`
SquashConsecRunesAll would return `fo` for all of them.
SquashConsecRunesAll will be a no-op and return an empty string
if `s` is an empty string.
*/
func SquashConsecRunesAll(s string) (squashed string) {
var r rune
var runeSz int
var prev rune = -1
var sb *strings.Builder
squashed = s
if len(s) == 0 {
return
}
sb = new(strings.Builder)
sb.Grow(len(s))
for idx := 0; idx < len(squashed); {
r, runeSz = utf8.DecodeRuneInString(squashed[idx:])
if r == utf8.RuneError && runeSz == 1 {
prev = -1
sb.WriteString(squashed[idx : idx+runeSz])
} else {
if prev != r {
prev = r
sb.WriteRune(r)
}
}
idx += runeSz
}
squashed = sb.String()
return
}
/*
SquashMap offers replacement of consecutive string series.
It is expected that s is UTF-8 or a compatible subset (e.g. ASCII).
If single is set to true, *single/standalone* instances of a sequence
in replaceMap keys will also be replaced with its corresponding value.
Be aware that this function is not ideal for very large replaceMap maps
or for very large strings, as s will be passed over for each key in replaceMap.
If dealing with very large strings or a very large explicit set of replacements,
you may want to directly use [strings.Map] or [strings.NewReplacer] (if multi-rune
replacement is needed) with your own logic.
Matching string replacements are done longest-to-shortest sequentially.
This means that given the string `fooo` (and assuming single==true),
and a replaceMap of:
map[string]string{
"o": "a",
"oo": "b",
}
the result will be "fba", not:
* "faaa"
* "fab"
Thus you may want to execute several calls if this is undesired.
Keys that are the same length are ordered alphabetically/alphanumerically
so this function is deterministic.
Empty keys in replaceMap (but NOT empty values) are skipped.
Keys in replaceMap that are not UTF-8 are skipped.
Note that this function offers high-granularity control over replacements
at the cost of complex setup.
If you want more simple squashing/collapsing of whitespace, [SquashWhitespace]
may be more apropos.
If you want more simple squashing/collapsing of non-whitespace, [SquashConsec]
may be more apropos.
If replaceMap is nil or empty, s will be returned as-is.
If s is an empty string, SquashMap will no-op.
*/
func SquashMap(s string, replaceMap map[string]string, single bool) (squashed string) {
var idx int
var sIdx int
var k string
var v string
var start int
var nextNum int
var nextPos int
var sb *strings.Builder
type kLen struct {
// k is the Key name.
k string
// r is the replacement string.
r string
// l is the length of k (in UTF-8 runes).
l int
// bl is the length of k (in bytes).
bl int
}
var kl kLen
var keysByLen []kLen
squashed = s
if len(replaceMap) == 0 || len(s) == 0 {
return
}
keysByLen = make([]kLen, 0, len(replaceMap))
for k, v = range replaceMap {
if k == "" {
continue
}
if !utf8.ValidString(k) {
continue
}
keysByLen = append(
keysByLen,
kLen{
k: k,
r: v,
l: utf8.RuneCountInString(k),
bl: len(k),
},
)
}
slices.SortFunc(
keysByLen,
func(a, b kLen) (cmp int) {
cmp = 0
// Sort so that the longer comes first.
if a.l > b.l {
cmp = -1
} else if a.l < b.l {
cmp = 1
} else if a.l == b.l {
if a.k < b.k {
cmp = -1
} else if a.k > b.k {
cmp = 1
}
}
return
},
)
for _, kl = range keysByLen {
switch strings.Count(squashed, kl.k) {
case 0:
continue
case 1:
if !single {
continue
}
squashed = strings.ReplaceAll(squashed, kl.k, kl.r)
default:
sb = new(strings.Builder)
sb.Grow(len(squashed))
for idx = 0; idx < len(squashed); {
if sIdx = strings.Index(squashed[idx:], kl.k); sIdx < 0 {
sb.WriteString(squashed[idx:])
break
}
start = idx + sIdx
sb.WriteString(squashed[idx:start])
nextNum = 0
nextPos = start
for strings.HasPrefix(squashed[nextPos:], kl.k) {
nextNum++
nextPos += kl.bl
}
if nextNum > 1 || (nextNum == 1 && single) {
sb.WriteString(kl.r)
} else {
sb.WriteString(kl.k)
}
idx = nextPos
}
squashed = sb.String()
}
}
return
}
/*
SquashWhitespace is used to collapse/squash consecutive whitespace.
It is expected that s is unicode or a compatible subset (e.g. ASCII).
If single is set to true, *single/standalone* instances of whitespace
will also be replaced with ws.
ws is a string that should replace *consecutive* whitespace.
ws does not necessarily have to be a whitespace string, and may be empty.
If you are trying to squash/collapse non-whitespace, [SquashConsec]
is more apropos.
If you want more fine-grained control over sequence replacement, see [SquashMap].
*/
func SquashWhitespace(s, ws string, single bool) (squashed string) {
var r rune
var runeSz int
var wsStart int
var invalid bool
var prevWs uint64
var sb *strings.Builder
squashed = s
if s == "" {
return
}
sb = new(strings.Builder)
sb.Grow(len(s))
for rIdx := 0; rIdx < len(s); {
r, runeSz = utf8.DecodeRuneInString(s[rIdx:])
invalid = r == utf8.RuneError && runeSz == 1
if invalid || !unicode.IsSpace(r) {
if prevWs > 1 || (prevWs == 1 && single) {
sb.WriteString(ws)
} else if prevWs == 1 {
sb.WriteString(s[wsStart:rIdx])
}
prevWs = 0
if invalid {
// Invalid UTF-8 rune; write byte as-is and advance index.
sb.WriteString(s[rIdx : rIdx+runeSz])
} else {
sb.WriteRune(r)
}
rIdx += runeSz
continue
}
if prevWs == 0 {
wsStart = rIdx
}
prevWs++
rIdx += runeSz
}
if prevWs > 1 || (prevWs == 1 && single) {
sb.WriteString(ws)
} else if prevWs == 1 {
sb.WriteString(s[wsStart:])
}
squashed = sb.String()
return
}
/*
StripWhitespace removes all leading, trailing, and INNER whitespace
from unicode string `s`.
+1 -1
View File
@@ -636,7 +636,7 @@ pre.rouge .gs {
<div class="details">
<span id="author" class="author">Brent Saner</span><br>
<span id="email" class="email"><a href="mailto:bts@square-r00t.net">bts@square-r00t.net</a></span><br>
<span id="revdate">Last rendered 2026-07-06 16:21:08 -0400</span>
<span id="revdate">Last rendered 2026-07-13 19:17:39 -0400</span>
</div>
<div id="toc" class="toc2">
<div id="toctitle">Table of Contents</div>
+1 -1
View File
@@ -6,7 +6,7 @@
<span id="author" class="author">Brent Saner</span>
<span id="email" class="email"><bts@square-r00t.net></span>
<span id="revdate">Last rendered 2026-07-06 16:21:13 -0400</span>
<span id="revdate">Last rendered 2026-07-13 19:17:43 -0400</span>
</div>