c9f3e7a639
ADDED: * stringsx: ** RunesInString ** RuneMapFromString ** SquashConsec ** SquashConsecRunes ** SquashConsecRunesAll ** SquashMap ** SquashWhitespace
1112 lines
25 KiB
Go
1112 lines
25 KiB
Go
package stringsx
|
|
|
|
import (
|
|
"bytes"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"slices"
|
|
"strings"
|
|
"unicode"
|
|
`unicode/utf8`
|
|
)
|
|
|
|
/*
|
|
HasBookend returns true if string s both begins AND ends with sym.
|
|
|
|
It is more strict than [HasBoundary] (which only requires
|
|
that s has sym at the beginning OR the end.)
|
|
|
|
Examples:
|
|
|
|
HasBookend("|foo|", "|") → true
|
|
HasBookend("|foo", "|") → false
|
|
HasBookend("foo|", "|") → false
|
|
HasBookend("fo|o", "|") → false
|
|
HasBookend("|foo| ", "|") → false // Whitespace prevents match
|
|
HasBookend(" |foo| ", "|") → false
|
|
|
|
sym may be a multi-rune string.
|
|
If sym is empty, HasBookend will *always* return true.
|
|
*/
|
|
func HasBookend(s, sym string) (bounded bool) {
|
|
|
|
bounded = strings.HasPrefix(s, sym) && strings.HasSuffix(s, sym)
|
|
|
|
return
|
|
}
|
|
|
|
/*
|
|
HasBoundary returns true if string s starts OR ends with symbol sym.
|
|
|
|
Examples:
|
|
|
|
HasBoundary("|foo|", "|") → true
|
|
HasBoundary("|foo", "|") → true
|
|
HasBoundary("foo|", "|") → true
|
|
HasBoundary("fo|o", "|") → false
|
|
HasBoundary("|foo| ", "|") → true
|
|
HasBoundary(" |foo| ", "|") → false // Whitespace prevents match
|
|
|
|
sym may be a multi-rune string.
|
|
If sym is empty, HasBoundary will *always* return true.
|
|
|
|
If you instead require string s to be *enclosed* by sym, see [HasBookend].
|
|
*/
|
|
func HasBoundary(s, sym string) (bounded bool) {
|
|
|
|
bounded = strings.HasPrefix(s, sym) || strings.HasSuffix(s, sym)
|
|
|
|
return
|
|
}
|
|
|
|
/*
|
|
IsAscii returns true if all characters in string s are ASCII.
|
|
|
|
This simply wraps [IsAsciiSpecial]:
|
|
|
|
isAscii, err = IsAsciiSpecial(s, allowCtl, true, allowExt, true, nil, nil)
|
|
*/
|
|
func IsAscii(s string, allowCtl, allowExt bool) (isAscii bool, err error) {
|
|
|
|
if isAscii, err = IsAsciiSpecial(
|
|
s, allowCtl, true, allowExt, true, nil, nil,
|
|
); err != nil {
|
|
return
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
/*
|
|
IsAsciiBuf returns true if all of buffer buf is valid ASCII.
|
|
|
|
Note that the buffer will be consumed/read by this function.
|
|
|
|
This simply wraps [IsAsciiBufSpecial]:
|
|
|
|
isAscii, err = IsAsciiBufSpecial(r, allowCtl, true, allowExt, true, nil, nil)
|
|
*/
|
|
func IsAsciiBuf(r io.RuneReader, allowCtl, allowExt bool) (isAscii bool, err error) {
|
|
|
|
if isAscii, err = IsAsciiBufSpecial(
|
|
r, allowCtl, true, allowExt, true, nil, nil,
|
|
); err != nil {
|
|
return
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
/*
|
|
IsAsciiSpecial allows for specifying specific ASCII ranges.
|
|
|
|
allowCtl, if true, will allow control characters (0x00 to 0x1f inclusive).
|
|
|
|
allowPrint, if true, will allow printable characters (what most people think of
|
|
when they say "ASCII") (0x20 to 0x7f inclusive).
|
|
|
|
allowExt, if true, will allow for "extended ASCII" - some later dialects expand
|
|
to a full 8-bit ASCII range (0x80 to 0xff inclusive).
|
|
|
|
wsCtl, if true, "shifts" the "whitespace control characters" (\t, \n, \r) to the "printable" space
|
|
(such that allowPrint controls their validation). Thus:
|
|
|
|
IsAsciiSpecial(s, false, true, false, true, nil, nil)
|
|
|
|
has the same effect as specifying:
|
|
|
|
IsAsciiSpecial(s, false, true, false, (-), []byte("\t\n\r"), nil)
|
|
|
|
incl, if non-nil and non-empty, allows *additional* characters to be specified as included
|
|
that would normally *not* be allowed.
|
|
|
|
excl, if non-nil and non-empty, invalidates on additional characters that would normally be allowed.
|
|
|
|
excl, if specified, takes precedence over incl if specified.
|
|
|
|
An [AsciiInvalidError] will be returned on the first encountered invalid character.
|
|
*/
|
|
func IsAsciiSpecial(s string, allowCtl, allowPrint, allowExt, allowWs bool, incl, excl []byte) (isAscii bool, err error) {
|
|
|
|
var buf *bytes.Buffer = bytes.NewBufferString(s)
|
|
|
|
if isAscii, err = IsAsciiBufSpecial(buf, allowCtl, allowPrint, allowExt, allowWs, incl, excl); err != nil {
|
|
return
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
/*
|
|
IsAsciiBufSpecial is the same as [IsAsciiSpecial] but operates on an [io.RuneReader].
|
|
|
|
Note that the buffer will be consumed/read by this function.
|
|
|
|
It will not return an [io.EOF] if encountered, but any other errors encountered will be returned.
|
|
It is expected that r will return an [io.EOF] when exhausted.
|
|
|
|
An [AsciiInvalidError] will be returned on the first encountered invalid character.
|
|
*/
|
|
func IsAsciiBufSpecial(r io.RuneReader, allowCtl, allowPrint, allowExt, allowWs bool, incl, excl []byte) (isAscii bool, err error) {
|
|
|
|
var b rune
|
|
var bLen int
|
|
var nextNewline bool
|
|
var tmpErr *AsciiInvalidError = new(AsciiInvalidError)
|
|
// I know, I know. This is essentually a lookup table. Keeps it speedy.
|
|
var allowed [256]bool = getAsciiCharMap(allowCtl, allowPrint, allowExt, allowWs, incl, excl)
|
|
|
|
for {
|
|
if b, bLen, err = r.ReadRune(); err != nil {
|
|
if errors.Is(err, io.EOF) {
|
|
err = nil
|
|
isAscii = true
|
|
}
|
|
return
|
|
}
|
|
// Set these *before* OK
|
|
if nextNewline {
|
|
tmpErr.Line++
|
|
tmpErr.LineByte = 0
|
|
tmpErr.LineChar = 0
|
|
nextNewline = false
|
|
} else {
|
|
tmpErr.LineChar++
|
|
}
|
|
tmpErr.Char++
|
|
|
|
if b == '\n' {
|
|
nextNewline = true
|
|
}
|
|
if b == rune(0xfffd) {
|
|
// not even valid unicode
|
|
tmpErr.BadChar = b
|
|
tmpErr.BadBytes = []byte(string(b))
|
|
err = tmpErr
|
|
return
|
|
}
|
|
if bLen > 2 || b > 0xff {
|
|
// ASCII only occupies a single byte, ISO-8859-1 occupies 2
|
|
tmpErr.BadChar = b
|
|
tmpErr.BadBytes = []byte(string(b))
|
|
err = tmpErr
|
|
return
|
|
}
|
|
if !allowed[byte(b)] {
|
|
tmpErr.BadChar = b
|
|
tmpErr.BadBytes = []byte{byte(b)}
|
|
err = tmpErr
|
|
return
|
|
}
|
|
|
|
// Set these *after* OK
|
|
tmpErr.LineByte += uint64(bLen)
|
|
tmpErr.Byte += uint64(bLen)
|
|
}
|
|
|
|
isAscii = true
|
|
|
|
return
|
|
}
|
|
|
|
/*
|
|
LenSplit formats string `s` to break at, at most, every `width` characters.
|
|
|
|
Any existing newlines (e.g. \r\n) will be removed during a string/
|
|
substring/line's length calculation. (e.g. `foobarbaz\n` and `foobarbaz\r\n` are
|
|
both considered to be lines of length 9, not 10 and 11 respectively).
|
|
|
|
This also means that any newlines (\n or \r\n) are inherently removed from
|
|
`out` (even if included in `wordWrap`; see below).
|
|
|
|
Note that if `s` is multiline (already contains newlines), they will be respected
|
|
as-is - that is, if a line ends with less than `width` chars and then has a newline,
|
|
it will be preserved as an empty element. That is to say:
|
|
|
|
"foo\nbar\n\n" → []string{"foo", "bar", ""}
|
|
"foo\n\nbar\n" → []string{"foo", "", "bar"}
|
|
|
|
This splitter is particularly simple. If you need wordwrapping, it should be done
|
|
with e.g. [github.com/muesli/reflow/wordwrap].
|
|
*/
|
|
func LenSplit(s string, width uint) (out []string) {
|
|
|
|
var end int
|
|
var line string
|
|
var lineRunes []rune
|
|
|
|
if width == 0 {
|
|
out = []string{s}
|
|
return
|
|
}
|
|
|
|
for line = range strings.Lines(s) {
|
|
line = strings.TrimRight(line, "\n")
|
|
line = strings.TrimRight(line, "\r")
|
|
|
|
lineRunes = []rune(line)
|
|
|
|
if uint(len(lineRunes)) <= width {
|
|
out = append(out, line)
|
|
continue
|
|
}
|
|
|
|
for i := 0; i < len(lineRunes); i += int(width) {
|
|
end = i + int(width)
|
|
if end > len(lineRunes) {
|
|
end = len(lineRunes)
|
|
}
|
|
out = append(out, string(lineRunes[i:end]))
|
|
}
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
/*
|
|
LenSplitStr wraps [LenSplit] but recombines into a new string with newlines.
|
|
|
|
It's mostly just a convenience wrapper.
|
|
|
|
All arguments remain the same as in [LenSplit] with an additional one,
|
|
`winNewLine`, which if true will use \r\n as the newline instead of \n.
|
|
*/
|
|
func LenSplitStr(s string, width uint, winNewline bool) (out string) {
|
|
|
|
var outSl []string = LenSplit(s, width)
|
|
|
|
if winNewline {
|
|
out = strings.Join(outSl, "\r\n")
|
|
} else {
|
|
out = strings.Join(outSl, "\n")
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
/*
|
|
Pad pads each element in `s` to length `width` using `pad`.
|
|
If `pad` is empty, a single space (0x20) will be assumed.
|
|
Note that `width` operates on rune size, not byte size.
|
|
(In ASCII, they will be the same size.)
|
|
|
|
If a line in `s` is greater than or equal to `width`,
|
|
no padding will be performed.
|
|
|
|
If `leftPad` is true, padding will be applied to the "left" (beginning")
|
|
of each element instead of the "right" ("end").
|
|
*/
|
|
func Pad(s []string, width uint, pad string, leftPad bool) (out []string) {
|
|
|
|
var idx int
|
|
var padIdx int
|
|
var runeIdx int
|
|
var padLen uint
|
|
var elem string
|
|
var unpadLen uint
|
|
var tmpPadLen int
|
|
var padRunes []rune
|
|
var tmpPad []rune
|
|
|
|
if width == 0 {
|
|
out = s
|
|
return
|
|
}
|
|
|
|
out = make([]string, len(s))
|
|
|
|
// Easy; supported directly in fmt.
|
|
if pad == "" {
|
|
for idx, elem = range s {
|
|
if leftPad {
|
|
out[idx] = fmt.Sprintf("%*s", width, elem)
|
|
} else {
|
|
out[idx] = fmt.Sprintf("%-*s", width, elem)
|
|
}
|
|
}
|
|
return
|
|
}
|
|
|
|
// This gets a little more tricky.
|
|
padRunes = []rune(pad)
|
|
padLen = uint(len(padRunes))
|
|
for idx, elem = range s {
|
|
// First we need to know the number of runes in elem.
|
|
unpadLen = uint(len([]rune(elem)))
|
|
// If it's more than/equal to width, as-is.
|
|
if unpadLen >= width {
|
|
out[idx] = elem
|
|
} else {
|
|
// Otherwise, we need to construct/calculate a pad.
|
|
if (width-unpadLen)%padLen == 0 {
|
|
// Also easy enough.
|
|
if leftPad {
|
|
out[idx] = fmt.Sprintf("%s%s", strings.Repeat(pad, int((width-unpadLen)/padLen)), elem)
|
|
} else {
|
|
out[idx] = fmt.Sprintf("%s%s", elem, strings.Repeat(pad, int((width-unpadLen)/padLen)))
|
|
}
|
|
} else {
|
|
// This is where it gets a little hairy.
|
|
tmpPad = []rune{}
|
|
tmpPadLen = int(width - unpadLen)
|
|
idx = 0
|
|
padIdx = 0
|
|
for runeIdx = range tmpPadLen {
|
|
tmpPad[runeIdx] = padRunes[padIdx]
|
|
if uint(padIdx) >= padLen {
|
|
padIdx = 0
|
|
} else {
|
|
padIdx++
|
|
}
|
|
runeIdx++
|
|
}
|
|
if leftPad {
|
|
out[idx] = fmt.Sprintf("%s%s", string(tmpPad), elem)
|
|
} else {
|
|
out[idx] = fmt.Sprintf("%s%s", elem, string(tmpPad))
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
/*
|
|
Redact provides a "masked" version of string s (e.g. `my_terrible_password` -> `my****************rd`).
|
|
|
|
maskStr is the character or sequence of characters
|
|
to repeat for every masked character of s.
|
|
If an empty string, the default [DefMaskStr] will be used.
|
|
(maskStr does not need to be a single character.
|
|
It is recommended to use a multi-char mask to help obfuscate a string's length.)
|
|
|
|
leading specifies the number of leading characters of s to leave *unmasked*.
|
|
If 0, no leading characters will be unmasked.
|
|
|
|
trailing specifies the number of trailing characters of s to leave *unmasked*.
|
|
if 0, no trailing characters will be unmasked.
|
|
|
|
newlines, if true, will preserve newline characters - otherwise
|
|
they will be treated as regular characters.
|
|
|
|
As a safety precaution, if:
|
|
|
|
len(s) <= (leading + trailing)
|
|
|
|
then the entire string will be *masked* and no unmasking will be performed.
|
|
|
|
Note that this DOES NOT do a string *replace*, it provides a masked version of `s` itself.
|
|
Wrap Redact with [strings.ReplaceAll] if you want to replace a certain value with a masked one.
|
|
*/
|
|
func Redact(s, maskStr string, leading, trailing uint, newlines bool) (redacted string) {
|
|
|
|
var nl string
|
|
var numMasked int
|
|
var sb strings.Builder
|
|
var endIdx int = int(leading)
|
|
|
|
// This condition functionally won't do anything, so just return the input as-is.
|
|
if s == "" {
|
|
return
|
|
}
|
|
|
|
if maskStr == "" {
|
|
maskStr = DefMaskStr
|
|
}
|
|
|
|
if newlines {
|
|
for line := range strings.Lines(s) {
|
|
nl = getNewLine(line)
|
|
sb.WriteString(
|
|
Redact(
|
|
strings.TrimSuffix(line, nl), maskStr, leading, trailing, false,
|
|
),
|
|
)
|
|
sb.WriteString(nl)
|
|
}
|
|
} else {
|
|
if len(s) <= int(leading+trailing) {
|
|
redacted = strings.Repeat(maskStr, len(s))
|
|
return
|
|
}
|
|
|
|
if leading == 0 && trailing == 0 {
|
|
redacted = strings.Repeat(maskStr, len(s))
|
|
return
|
|
}
|
|
|
|
numMasked = len(s) - int(leading+trailing)
|
|
endIdx = endIdx + numMasked
|
|
|
|
if leading > 0 {
|
|
sb.WriteString(s[:int(leading)])
|
|
}
|
|
|
|
sb.WriteString(strings.Repeat(maskStr, numMasked))
|
|
|
|
if trailing > 0 {
|
|
sb.WriteString(s[endIdx:])
|
|
}
|
|
}
|
|
|
|
redacted = sb.String()
|
|
|
|
return
|
|
}
|
|
|
|
/*
|
|
RemoveWhitespace removes all leading, trailing, and INNER whitespace
|
|
from unicode string `s`.
|
|
|
|
This is done by allocating a strings.Builder with len(s).
|
|
This may consume more memory than needed if s is mostly whitespace;
|
|
in that case, it is better to use [StripWhitespace].
|
|
*/
|
|
func RemoveWhitespace(s string) (removed string) {
|
|
|
|
// https://stackoverflow.com/questions/32081808/strip-all-whitespace-from-a-string/32081891#32081891
|
|
|
|
var c rune
|
|
var sb strings.Builder
|
|
|
|
if s == "" {
|
|
return
|
|
}
|
|
|
|
sb.Grow(len(s))
|
|
for _, c = range s {
|
|
if !unicode.IsSpace(c) {
|
|
sb.WriteRune(c)
|
|
}
|
|
}
|
|
|
|
removed = sb.String()
|
|
|
|
return
|
|
}
|
|
|
|
// Reverse reverses string s. (It's absolutely insane that this isn't in stdlib.)
|
|
func Reverse(s string) (revS string) {
|
|
|
|
var rsl []rune = []rune(s)
|
|
|
|
slices.Reverse(rsl)
|
|
|
|
revS = string(rsl)
|
|
|
|
return
|
|
}
|
|
|
|
/*
|
|
RunesInString returns an ordered slice of all unique runes in
|
|
UTF-8 string `s`.
|
|
|
|
If `s` is an empty string, runes will be a non-nil empty slice.
|
|
*/
|
|
func RunesInString(s string) (runes []rune) {
|
|
|
|
var r rune
|
|
var idx int
|
|
var runeMap map[rune]uint64 = RuneMapFromString(s)
|
|
|
|
runes = make([]rune, len(runeMap))
|
|
|
|
idx = 0
|
|
for r, _ = range runeMap {
|
|
runes[idx] = r
|
|
idx++
|
|
}
|
|
|
|
slices.Sort(runes)
|
|
|
|
return
|
|
}
|
|
|
|
/*
|
|
RuneMapFromString returns a map of unique runes found in
|
|
UTF-8 string `s` along with a count of their occurrence.
|
|
|
|
If `s` is an empty string, runes will be a non-nil empty map.
|
|
|
|
Non-UTF-8 runes are skipped.
|
|
*/
|
|
func RuneMapFromString(s string) (runes map[rune]uint64) {
|
|
|
|
var r rune
|
|
var runeSz int
|
|
|
|
runes = make(map[rune]uint64)
|
|
|
|
for idx := 0; idx < len(s); {
|
|
r, runeSz = utf8.DecodeRuneInString(s[idx:])
|
|
if r == utf8.RuneError && runeSz == 1 {
|
|
idx += runeSz
|
|
continue
|
|
}
|
|
runes[r]++
|
|
idx += runeSz
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
/*
|
|
SquashConsec squashes/collapses consecutive instances of sequences
|
|
seq to a single instance.
|
|
It is expected that s is UTF-8 or a compatible subset (e.g. ASCII).
|
|
|
|
If seq is nil/empty, any n+1 consecutive instance of *any rune*
|
|
will be squashed to a single instance. This is *much* more performant
|
|
(as it simply wraps [SquashConsecRunesAll]) at the cost of lack of scoping.
|
|
|
|
If you are trying to squash/collapse whitespace instead, [SquashWhitespace]
|
|
may be more apropos.
|
|
|
|
If you want more fine-grained control over sequence replacement, see [SquashMap].
|
|
|
|
If s is an empty string, SquashConsec will return `s` as-is.
|
|
*/
|
|
func SquashConsec(s string, seq ...string) (squashed string) {
|
|
|
|
var idx int
|
|
var sIdx int
|
|
var kLen int
|
|
var start int
|
|
var seqIdx int
|
|
var nextNum int
|
|
var nextPos int
|
|
var sb *strings.Builder
|
|
|
|
squashed = s
|
|
|
|
if len(s) == 0 {
|
|
return
|
|
}
|
|
|
|
if len(seq) == 0 {
|
|
squashed = SquashConsecRunesAll(s)
|
|
return
|
|
}
|
|
|
|
for seqIdx = range seq {
|
|
if seq[seqIdx] == "" {
|
|
continue
|
|
}
|
|
sb = new(strings.Builder)
|
|
sb.Grow(len(squashed))
|
|
kLen = len(seq[seqIdx])
|
|
for idx = 0; idx < len(squashed); {
|
|
sIdx = strings.Index(squashed[idx:], seq[seqIdx])
|
|
if sIdx < 0 {
|
|
sb.WriteString(squashed[idx:])
|
|
break
|
|
}
|
|
start = idx + sIdx
|
|
sb.WriteString(squashed[idx:start])
|
|
nextNum = 0
|
|
nextPos = start
|
|
for strings.HasPrefix(squashed[nextPos:], seq[seqIdx]) {
|
|
nextNum++
|
|
nextPos += kLen
|
|
}
|
|
if nextNum >= 1 {
|
|
sb.WriteString(seq[seqIdx])
|
|
}
|
|
idx = nextPos
|
|
}
|
|
|
|
squashed = sb.String()
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
/*
|
|
SquashConsecRunes squashes/collapses consecutive instances of sequences
|
|
seq with a single instance.
|
|
It is expected that s is UTF-8 or a compatible subset (e.g. ASCII).
|
|
|
|
If seq is nil/empty, any n+1 consecutive instance of *any rune*
|
|
will be squashed to a single instance. This is *much* more performant
|
|
(as it simply wraps [SquashConsecRunesAll]) at the cost of lack of scoping.
|
|
|
|
If s is an empty string, SquashConsecRunes will return `s` as-is.
|
|
*/
|
|
func SquashConsecRunes(s string, seq ...rune) (squashed string) {
|
|
|
|
var r rune
|
|
var idx int
|
|
var prev rune
|
|
var seqIdx int
|
|
var runeSz int
|
|
var sb *strings.Builder
|
|
|
|
squashed = s
|
|
|
|
if len(s) == 0 {
|
|
return
|
|
}
|
|
|
|
if len(seq) == 0 {
|
|
squashed = SquashConsecRunesAll(s)
|
|
return
|
|
}
|
|
|
|
for seqIdx = range seq {
|
|
if seq[seqIdx] < 0 {
|
|
continue
|
|
}
|
|
sb = new(strings.Builder)
|
|
sb.Grow(len(squashed))
|
|
prev = -1
|
|
for idx = 0; idx < len(squashed); {
|
|
r, runeSz = utf8.DecodeRuneInString(squashed[idx:])
|
|
if r == utf8.RuneError && runeSz == 1 {
|
|
prev = -1
|
|
sb.WriteString(squashed[idx : idx+runeSz])
|
|
} else {
|
|
if r != seq[seqIdx] || prev != r {
|
|
prev = r
|
|
sb.WriteRune(r)
|
|
}
|
|
}
|
|
idx += runeSz
|
|
}
|
|
|
|
squashed = sb.String()
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
/*
|
|
SquashConsecRunesAll squashes/condenses/collapses all consecutive occurrences
|
|
of every rune in `s` to a single instance.
|
|
It is expected that s is UTF-8 or a compatible subset (e.g. ASCII);
|
|
non-UTF-8 runes will be written as-is to squashed (even if duplicate).
|
|
|
|
That is to say, for an `s` of:
|
|
|
|
* `fo`
|
|
* `foo`
|
|
* `fooo`
|
|
* `foooo`
|
|
|
|
SquashConsecRunesAll would return `fo` for all of them.
|
|
|
|
SquashConsecRunesAll will be a no-op and return an empty string
|
|
if `s` is an empty string.
|
|
*/
|
|
func SquashConsecRunesAll(s string) (squashed string) {
|
|
|
|
var r rune
|
|
var runeSz int
|
|
var prev rune = -1
|
|
var sb *strings.Builder
|
|
|
|
squashed = s
|
|
|
|
if len(s) == 0 {
|
|
return
|
|
}
|
|
|
|
sb = new(strings.Builder)
|
|
sb.Grow(len(s))
|
|
for idx := 0; idx < len(squashed); {
|
|
r, runeSz = utf8.DecodeRuneInString(squashed[idx:])
|
|
if r == utf8.RuneError && runeSz == 1 {
|
|
prev = -1
|
|
sb.WriteString(squashed[idx : idx+runeSz])
|
|
} else {
|
|
if prev != r {
|
|
prev = r
|
|
sb.WriteRune(r)
|
|
}
|
|
}
|
|
idx += runeSz
|
|
}
|
|
|
|
squashed = sb.String()
|
|
|
|
return
|
|
}
|
|
|
|
/*
|
|
SquashMap offers replacement of consecutive string series.
|
|
It is expected that s is UTF-8 or a compatible subset (e.g. ASCII).
|
|
|
|
If single is set to true, *single/standalone* instances of a sequence
|
|
in replaceMap keys will also be replaced with its corresponding value.
|
|
|
|
Be aware that this function is not ideal for very large replaceMap maps
|
|
or for very large strings, as s will be passed over for each key in replaceMap.
|
|
If dealing with very large strings or a very large explicit set of replacements,
|
|
you may want to directly use [strings.Map] or [strings.NewReplacer] (if multi-rune
|
|
replacement is needed) with your own logic.
|
|
|
|
Matching string replacements are done longest-to-shortest sequentially.
|
|
This means that given the string `fooo` (and assuming single==true),
|
|
and a replaceMap of:
|
|
|
|
map[string]string{
|
|
"o": "a",
|
|
"oo": "b",
|
|
}
|
|
|
|
the result will be "fba", not:
|
|
|
|
* "faaa"
|
|
* "fab"
|
|
|
|
Thus you may want to execute several calls if this is undesired.
|
|
Keys that are the same length are ordered alphabetically/alphanumerically
|
|
so this function is deterministic.
|
|
|
|
Empty keys in replaceMap (but NOT empty values) are skipped.
|
|
Keys in replaceMap that are not UTF-8 are skipped.
|
|
|
|
Note that this function offers high-granularity control over replacements
|
|
at the cost of complex setup.
|
|
|
|
If you want more simple squashing/collapsing of whitespace, [SquashWhitespace]
|
|
may be more apropos.
|
|
|
|
If you want more simple squashing/collapsing of non-whitespace, [SquashConsec]
|
|
may be more apropos.
|
|
|
|
If replaceMap is nil or empty, s will be returned as-is.
|
|
If s is an empty string, SquashMap will no-op.
|
|
*/
|
|
func SquashMap(s string, replaceMap map[string]string, single bool) (squashed string) {
|
|
|
|
var idx int
|
|
var sIdx int
|
|
var k string
|
|
var v string
|
|
var start int
|
|
var nextNum int
|
|
var nextPos int
|
|
var sb *strings.Builder
|
|
type kLen struct {
|
|
// k is the Key name.
|
|
k string
|
|
// r is the replacement string.
|
|
r string
|
|
// l is the length of k (in UTF-8 runes).
|
|
l int
|
|
// bl is the length of k (in bytes).
|
|
bl int
|
|
}
|
|
var kl kLen
|
|
var keysByLen []kLen
|
|
|
|
squashed = s
|
|
|
|
if len(replaceMap) == 0 || len(s) == 0 {
|
|
return
|
|
}
|
|
|
|
keysByLen = make([]kLen, 0, len(replaceMap))
|
|
for k, v = range replaceMap {
|
|
if k == "" {
|
|
continue
|
|
}
|
|
if !utf8.ValidString(k) {
|
|
continue
|
|
}
|
|
keysByLen = append(
|
|
keysByLen,
|
|
kLen{
|
|
k: k,
|
|
r: v,
|
|
l: utf8.RuneCountInString(k),
|
|
bl: len(k),
|
|
},
|
|
)
|
|
}
|
|
slices.SortFunc(
|
|
keysByLen,
|
|
func(a, b kLen) (cmp int) {
|
|
cmp = 0
|
|
// Sort so that the longer comes first.
|
|
if a.l > b.l {
|
|
cmp = -1
|
|
} else if a.l < b.l {
|
|
cmp = 1
|
|
} else if a.l == b.l {
|
|
if a.k < b.k {
|
|
cmp = -1
|
|
} else if a.k > b.k {
|
|
cmp = 1
|
|
}
|
|
}
|
|
return
|
|
},
|
|
)
|
|
|
|
for _, kl = range keysByLen {
|
|
switch strings.Count(squashed, kl.k) {
|
|
case 0:
|
|
continue
|
|
case 1:
|
|
if !single {
|
|
continue
|
|
}
|
|
squashed = strings.ReplaceAll(squashed, kl.k, kl.r)
|
|
default:
|
|
sb = new(strings.Builder)
|
|
sb.Grow(len(squashed))
|
|
for idx = 0; idx < len(squashed); {
|
|
if sIdx = strings.Index(squashed[idx:], kl.k); sIdx < 0 {
|
|
sb.WriteString(squashed[idx:])
|
|
break
|
|
}
|
|
start = idx + sIdx
|
|
sb.WriteString(squashed[idx:start])
|
|
nextNum = 0
|
|
nextPos = start
|
|
for strings.HasPrefix(squashed[nextPos:], kl.k) {
|
|
nextNum++
|
|
nextPos += kl.bl
|
|
}
|
|
if nextNum > 1 || (nextNum == 1 && single) {
|
|
sb.WriteString(kl.r)
|
|
} else {
|
|
sb.WriteString(kl.k)
|
|
}
|
|
idx = nextPos
|
|
}
|
|
|
|
squashed = sb.String()
|
|
}
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
/*
|
|
SquashWhitespace is used to collapse/squash consecutive whitespace.
|
|
It is expected that s is unicode or a compatible subset (e.g. ASCII).
|
|
|
|
If single is set to true, *single/standalone* instances of whitespace
|
|
will also be replaced with ws.
|
|
|
|
ws is a string that should replace *consecutive* whitespace.
|
|
ws does not necessarily have to be a whitespace string, and may be empty.
|
|
|
|
If you are trying to squash/collapse non-whitespace, [SquashConsec]
|
|
is more apropos.
|
|
|
|
If you want more fine-grained control over sequence replacement, see [SquashMap].
|
|
*/
|
|
func SquashWhitespace(s, ws string, single bool) (squashed string) {
|
|
|
|
var r rune
|
|
var runeSz int
|
|
var wsStart int
|
|
var invalid bool
|
|
var prevWs uint64
|
|
var sb *strings.Builder
|
|
|
|
squashed = s
|
|
|
|
if s == "" {
|
|
return
|
|
}
|
|
|
|
sb = new(strings.Builder)
|
|
sb.Grow(len(s))
|
|
|
|
for rIdx := 0; rIdx < len(s); {
|
|
r, runeSz = utf8.DecodeRuneInString(s[rIdx:])
|
|
invalid = r == utf8.RuneError && runeSz == 1
|
|
if invalid || !unicode.IsSpace(r) {
|
|
if prevWs > 1 || (prevWs == 1 && single) {
|
|
sb.WriteString(ws)
|
|
} else if prevWs == 1 {
|
|
sb.WriteString(s[wsStart:rIdx])
|
|
}
|
|
prevWs = 0
|
|
if invalid {
|
|
// Invalid UTF-8 rune; write byte as-is and advance index.
|
|
sb.WriteString(s[rIdx : rIdx+runeSz])
|
|
} else {
|
|
sb.WriteRune(r)
|
|
}
|
|
rIdx += runeSz
|
|
continue
|
|
}
|
|
if prevWs == 0 {
|
|
wsStart = rIdx
|
|
}
|
|
prevWs++
|
|
rIdx += runeSz
|
|
}
|
|
if prevWs > 1 || (prevWs == 1 && single) {
|
|
sb.WriteString(ws)
|
|
} else if prevWs == 1 {
|
|
sb.WriteString(s[wsStart:])
|
|
}
|
|
|
|
squashed = sb.String()
|
|
|
|
return
|
|
}
|
|
|
|
/*
|
|
StripWhitespace removes all leading, trailing, and INNER whitespace
|
|
from unicode string `s`.
|
|
|
|
This is done by mapping each rune in `s`.
|
|
This may use far more allocations than necessary if `s` is mostly NON-whitespace;
|
|
in that case, it is better to use [RemoveWhitespace].
|
|
*/
|
|
func StripWhitespace(s string) (stripped string) {
|
|
|
|
// https://stackoverflow.com/questions/32081808/strip-all-whitespace-from-a-string/32081891#32081891
|
|
|
|
if s == "" {
|
|
return
|
|
}
|
|
|
|
stripped = strings.Map(
|
|
func(c rune) rune {
|
|
if unicode.IsSpace(c) {
|
|
return -1
|
|
}
|
|
return c
|
|
},
|
|
s,
|
|
)
|
|
|
|
return
|
|
}
|
|
|
|
/*
|
|
TrimLines is like [strings.TrimSpace] but operates on *each line* of s.
|
|
It is *NIX-newline (`\n`) vs. Windows-newline (`\r\n`) agnostic.
|
|
The first encountered linebreak (`\n` vs. `\r\n`) are assumed to be
|
|
the canonical linebreak for the rest of s.
|
|
|
|
left, if true, performs a [TrimSpaceLeft] on each line (retaining the newline).
|
|
|
|
right, if true, performs a [TrimSpaceRight] on each line (retaining the newline).
|
|
*/
|
|
func TrimLines(s string, left, right bool) (trimmed string) {
|
|
|
|
var sl string
|
|
var nl string
|
|
var sb strings.Builder
|
|
|
|
// These conditions functionally won't do anything, so just return the input as-is.
|
|
if s == "" {
|
|
return
|
|
}
|
|
if !left && !right {
|
|
trimmed = s
|
|
return
|
|
}
|
|
|
|
for line := range strings.Lines(s) {
|
|
nl = getNewLine(line)
|
|
sl = strings.TrimSuffix(line, nl)
|
|
if left && right {
|
|
sl = strings.TrimSpace(sl)
|
|
} else if left {
|
|
sl = TrimSpaceLeft(sl)
|
|
} else if right {
|
|
sl = TrimSpaceRight(sl)
|
|
}
|
|
sb.WriteString(sl)
|
|
sb.WriteString(nl)
|
|
}
|
|
|
|
trimmed = sb.String()
|
|
|
|
return
|
|
}
|
|
|
|
// TrimSpaceLeft is like [strings.TrimSpace] but only removes leading whitespace from string `s`.
|
|
func TrimSpaceLeft(s string) (trimmed string) {
|
|
|
|
trimmed = strings.TrimLeftFunc(s, unicode.IsSpace)
|
|
|
|
return
|
|
}
|
|
|
|
/*
|
|
TrimSpaceRight is like [strings.TrimSpace] but only removes trailing whitespace from string s.
|
|
*/
|
|
func TrimSpaceRight(s string) (trimmed string) {
|
|
|
|
trimmed = strings.TrimRightFunc(s, unicode.IsSpace)
|
|
|
|
return
|
|
}
|
|
|
|
// getAsciiCharMap returns a lookup "table" for ASCII characters.
|
|
func getAsciiCharMap(allowCtl, allowPrint, allowExt, allowWs bool, incl, excl []byte) (charmap [256]bool) {
|
|
|
|
var idx uint8
|
|
|
|
if allowCtl {
|
|
for idx < 0x1f {
|
|
charmap[idx] = true
|
|
idx++
|
|
}
|
|
} else {
|
|
idx = 0x1f
|
|
}
|
|
if allowPrint {
|
|
for idx < 0x7f {
|
|
charmap[idx] = true
|
|
idx++
|
|
}
|
|
} else {
|
|
idx = 0x7f
|
|
}
|
|
if allowExt {
|
|
for {
|
|
charmap[idx] = true
|
|
if idx == 0xff {
|
|
break
|
|
}
|
|
idx++
|
|
}
|
|
} else {
|
|
idx = 0xff
|
|
}
|
|
if allowWs {
|
|
charmap['\t'] = true
|
|
charmap['\n'] = true
|
|
charmap['\r'] = true
|
|
}
|
|
|
|
if incl != nil && len(incl) > 0 {
|
|
for _, idx = range incl {
|
|
charmap[idx] = true
|
|
}
|
|
}
|
|
if excl != nil && len(excl) > 0 {
|
|
for _, idx = range excl {
|
|
charmap[idx] = false
|
|
}
|
|
}
|
|
|
|
return
|
|
}
|
|
|
|
// getNewLine is too unpredictable/nuanced to be used as part of a public API promise so it isn't exported.
|
|
func getNewLine(s string) (nl string) {
|
|
|
|
if strings.HasSuffix(s, "\r\n") {
|
|
nl = "\r\n"
|
|
} else if strings.HasSuffix(s, "\n") {
|
|
nl = "\n"
|
|
}
|
|
|
|
return
|
|
}
|