package stringsx import ( "bytes" "errors" "fmt" "io" "slices" "strings" "unicode" `unicode/utf8` ) /* HasBookend returns true if string s both begins AND ends with sym. It is more strict than [HasBoundary] (which only requires that s has sym at the beginning OR the end.) Examples: HasBookend("|foo|", "|") → true HasBookend("|foo", "|") → false HasBookend("foo|", "|") → false HasBookend("fo|o", "|") → false HasBookend("|foo| ", "|") → false // Whitespace prevents match HasBookend(" |foo| ", "|") → false sym may be a multi-rune string. If sym is empty, HasBookend will *always* return true. */ func HasBookend(s, sym string) (bounded bool) { bounded = strings.HasPrefix(s, sym) && strings.HasSuffix(s, sym) return } /* HasBoundary returns true if string s starts OR ends with symbol sym. Examples: HasBoundary("|foo|", "|") → true HasBoundary("|foo", "|") → true HasBoundary("foo|", "|") → true HasBoundary("fo|o", "|") → false HasBoundary("|foo| ", "|") → true HasBoundary(" |foo| ", "|") → false // Whitespace prevents match sym may be a multi-rune string. If sym is empty, HasBoundary will *always* return true. If you instead require string s to be *enclosed* by sym, see [HasBookend]. */ func HasBoundary(s, sym string) (bounded bool) { bounded = strings.HasPrefix(s, sym) || strings.HasSuffix(s, sym) return } /* IsAscii returns true if all characters in string s are ASCII. This simply wraps [IsAsciiSpecial]: isAscii, err = IsAsciiSpecial(s, allowCtl, true, allowExt, true, nil, nil) */ func IsAscii(s string, allowCtl, allowExt bool) (isAscii bool, err error) { if isAscii, err = IsAsciiSpecial( s, allowCtl, true, allowExt, true, nil, nil, ); err != nil { return } return } /* IsAsciiBuf returns true if all of buffer buf is valid ASCII. Note that the buffer will be consumed/read by this function. This simply wraps [IsAsciiBufSpecial]: isAscii, err = IsAsciiBufSpecial(r, allowCtl, true, allowExt, true, nil, nil) */ func IsAsciiBuf(r io.RuneReader, allowCtl, allowExt bool) (isAscii bool, err error) { if isAscii, err = IsAsciiBufSpecial( r, allowCtl, true, allowExt, true, nil, nil, ); err != nil { return } return } /* IsAsciiSpecial allows for specifying specific ASCII ranges. allowCtl, if true, will allow control characters (0x00 to 0x1f inclusive). allowPrint, if true, will allow printable characters (what most people think of when they say "ASCII") (0x20 to 0x7f inclusive). allowExt, if true, will allow for "extended ASCII" - some later dialects expand to a full 8-bit ASCII range (0x80 to 0xff inclusive). wsCtl, if true, "shifts" the "whitespace control characters" (\t, \n, \r) to the "printable" space (such that allowPrint controls their validation). Thus: IsAsciiSpecial(s, false, true, false, true, nil, nil) has the same effect as specifying: IsAsciiSpecial(s, false, true, false, (-), []byte("\t\n\r"), nil) incl, if non-nil and non-empty, allows *additional* characters to be specified as included that would normally *not* be allowed. excl, if non-nil and non-empty, invalidates on additional characters that would normally be allowed. excl, if specified, takes precedence over incl if specified. An [AsciiInvalidError] will be returned on the first encountered invalid character. */ func IsAsciiSpecial(s string, allowCtl, allowPrint, allowExt, allowWs bool, incl, excl []byte) (isAscii bool, err error) { var buf *bytes.Buffer = bytes.NewBufferString(s) if isAscii, err = IsAsciiBufSpecial(buf, allowCtl, allowPrint, allowExt, allowWs, incl, excl); err != nil { return } return } /* IsAsciiBufSpecial is the same as [IsAsciiSpecial] but operates on an [io.RuneReader]. Note that the buffer will be consumed/read by this function. It will not return an [io.EOF] if encountered, but any other errors encountered will be returned. It is expected that r will return an [io.EOF] when exhausted. An [AsciiInvalidError] will be returned on the first encountered invalid character. */ func IsAsciiBufSpecial(r io.RuneReader, allowCtl, allowPrint, allowExt, allowWs bool, incl, excl []byte) (isAscii bool, err error) { var b rune var bLen int var nextNewline bool var tmpErr *AsciiInvalidError = new(AsciiInvalidError) // I know, I know. This is essentually a lookup table. Keeps it speedy. var allowed [256]bool = getAsciiCharMap(allowCtl, allowPrint, allowExt, allowWs, incl, excl) for { if b, bLen, err = r.ReadRune(); err != nil { if errors.Is(err, io.EOF) { err = nil isAscii = true } return } // Set these *before* OK if nextNewline { tmpErr.Line++ tmpErr.LineByte = 0 tmpErr.LineChar = 0 nextNewline = false } else { tmpErr.LineChar++ } tmpErr.Char++ if b == '\n' { nextNewline = true } if b == rune(0xfffd) { // not even valid unicode tmpErr.BadChar = b tmpErr.BadBytes = []byte(string(b)) err = tmpErr return } if bLen > 2 || b > 0xff { // ASCII only occupies a single byte, ISO-8859-1 occupies 2 tmpErr.BadChar = b tmpErr.BadBytes = []byte(string(b)) err = tmpErr return } if !allowed[byte(b)] { tmpErr.BadChar = b tmpErr.BadBytes = []byte{byte(b)} err = tmpErr return } // Set these *after* OK tmpErr.LineByte += uint64(bLen) tmpErr.Byte += uint64(bLen) } isAscii = true return } /* LenSplit formats string `s` to break at, at most, every `width` characters. Any existing newlines (e.g. \r\n) will be removed during a string/ substring/line's length calculation. (e.g. `foobarbaz\n` and `foobarbaz\r\n` are both considered to be lines of length 9, not 10 and 11 respectively). This also means that any newlines (\n or \r\n) are inherently removed from `out` (even if included in `wordWrap`; see below). Note that if `s` is multiline (already contains newlines), they will be respected as-is - that is, if a line ends with less than `width` chars and then has a newline, it will be preserved as an empty element. That is to say: "foo\nbar\n\n" → []string{"foo", "bar", ""} "foo\n\nbar\n" → []string{"foo", "", "bar"} This splitter is particularly simple. If you need wordwrapping, it should be done with e.g. [github.com/muesli/reflow/wordwrap]. */ func LenSplit(s string, width uint) (out []string) { var end int var line string var lineRunes []rune if width == 0 { out = []string{s} return } for line = range strings.Lines(s) { line = strings.TrimRight(line, "\n") line = strings.TrimRight(line, "\r") lineRunes = []rune(line) if uint(len(lineRunes)) <= width { out = append(out, line) continue } for i := 0; i < len(lineRunes); i += int(width) { end = i + int(width) if end > len(lineRunes) { end = len(lineRunes) } out = append(out, string(lineRunes[i:end])) } } return } /* LenSplitStr wraps [LenSplit] but recombines into a new string with newlines. It's mostly just a convenience wrapper. All arguments remain the same as in [LenSplit] with an additional one, `winNewLine`, which if true will use \r\n as the newline instead of \n. */ func LenSplitStr(s string, width uint, winNewline bool) (out string) { var outSl []string = LenSplit(s, width) if winNewline { out = strings.Join(outSl, "\r\n") } else { out = strings.Join(outSl, "\n") } return } /* Pad pads each element in `s` to length `width` using `pad`. If `pad` is empty, a single space (0x20) will be assumed. Note that `width` operates on rune size, not byte size. (In ASCII, they will be the same size.) If a line in `s` is greater than or equal to `width`, no padding will be performed. If `leftPad` is true, padding will be applied to the "left" (beginning") of each element instead of the "right" ("end"). */ func Pad(s []string, width uint, pad string, leftPad bool) (out []string) { var idx int var padIdx int var runeIdx int var padLen uint var elem string var unpadLen uint var tmpPadLen int var padRunes []rune var tmpPad []rune if width == 0 { out = s return } out = make([]string, len(s)) // Easy; supported directly in fmt. if pad == "" { for idx, elem = range s { if leftPad { out[idx] = fmt.Sprintf("%*s", width, elem) } else { out[idx] = fmt.Sprintf("%-*s", width, elem) } } return } // This gets a little more tricky. padRunes = []rune(pad) padLen = uint(len(padRunes)) for idx, elem = range s { // First we need to know the number of runes in elem. unpadLen = uint(len([]rune(elem))) // If it's more than/equal to width, as-is. if unpadLen >= width { out[idx] = elem } else { // Otherwise, we need to construct/calculate a pad. if (width-unpadLen)%padLen == 0 { // Also easy enough. if leftPad { out[idx] = fmt.Sprintf("%s%s", strings.Repeat(pad, int((width-unpadLen)/padLen)), elem) } else { out[idx] = fmt.Sprintf("%s%s", elem, strings.Repeat(pad, int((width-unpadLen)/padLen))) } } else { // This is where it gets a little hairy. tmpPad = []rune{} tmpPadLen = int(width - unpadLen) idx = 0 padIdx = 0 for runeIdx = range tmpPadLen { tmpPad[runeIdx] = padRunes[padIdx] if uint(padIdx) >= padLen { padIdx = 0 } else { padIdx++ } runeIdx++ } if leftPad { out[idx] = fmt.Sprintf("%s%s", string(tmpPad), elem) } else { out[idx] = fmt.Sprintf("%s%s", elem, string(tmpPad)) } } } } return } /* Redact provides a "masked" version of string s (e.g. `my_terrible_password` -> `my****************rd`). maskStr is the character or sequence of characters to repeat for every masked character of s. If an empty string, the default [DefMaskStr] will be used. (maskStr does not need to be a single character. It is recommended to use a multi-char mask to help obfuscate a string's length.) leading specifies the number of leading characters of s to leave *unmasked*. If 0, no leading characters will be unmasked. trailing specifies the number of trailing characters of s to leave *unmasked*. if 0, no trailing characters will be unmasked. newlines, if true, will preserve newline characters - otherwise they will be treated as regular characters. As a safety precaution, if: len(s) <= (leading + trailing) then the entire string will be *masked* and no unmasking will be performed. Note that this DOES NOT do a string *replace*, it provides a masked version of `s` itself. Wrap Redact with [strings.ReplaceAll] if you want to replace a certain value with a masked one. */ func Redact(s, maskStr string, leading, trailing uint, newlines bool) (redacted string) { var nl string var numMasked int var sb strings.Builder var endIdx int = int(leading) // This condition functionally won't do anything, so just return the input as-is. if s == "" { return } if maskStr == "" { maskStr = DefMaskStr } if newlines { for line := range strings.Lines(s) { nl = getNewLine(line) sb.WriteString( Redact( strings.TrimSuffix(line, nl), maskStr, leading, trailing, false, ), ) sb.WriteString(nl) } } else { if len(s) <= int(leading+trailing) { redacted = strings.Repeat(maskStr, len(s)) return } if leading == 0 && trailing == 0 { redacted = strings.Repeat(maskStr, len(s)) return } numMasked = len(s) - int(leading+trailing) endIdx = endIdx + numMasked if leading > 0 { sb.WriteString(s[:int(leading)]) } sb.WriteString(strings.Repeat(maskStr, numMasked)) if trailing > 0 { sb.WriteString(s[endIdx:]) } } redacted = sb.String() return } /* RemoveWhitespace removes all leading, trailing, and INNER whitespace from unicode string `s`. This is done by allocating a strings.Builder with len(s). This may consume more memory than needed if s is mostly whitespace; in that case, it is better to use [StripWhitespace]. */ func RemoveWhitespace(s string) (removed string) { // https://stackoverflow.com/questions/32081808/strip-all-whitespace-from-a-string/32081891#32081891 var c rune var sb strings.Builder if s == "" { return } sb.Grow(len(s)) for _, c = range s { if !unicode.IsSpace(c) { sb.WriteRune(c) } } removed = sb.String() return } // Reverse reverses string s. (It's absolutely insane that this isn't in stdlib.) func Reverse(s string) (revS string) { var rsl []rune = []rune(s) slices.Reverse(rsl) revS = string(rsl) return } /* RunesInString returns an ordered slice of all unique runes in UTF-8 string `s`. If `s` is an empty string, runes will be a non-nil empty slice. */ func RunesInString(s string) (runes []rune) { var r rune var idx int var runeMap map[rune]uint64 = RuneMapFromString(s) runes = make([]rune, len(runeMap)) idx = 0 for r, _ = range runeMap { runes[idx] = r idx++ } slices.Sort(runes) return } /* RuneMapFromString returns a map of unique runes found in UTF-8 string `s` along with a count of their occurrence. If `s` is an empty string, runes will be a non-nil empty map. Non-UTF-8 runes are skipped. */ func RuneMapFromString(s string) (runes map[rune]uint64) { var r rune var runeSz int runes = make(map[rune]uint64) for idx := 0; idx < len(s); { r, runeSz = utf8.DecodeRuneInString(s[idx:]) if r == utf8.RuneError && runeSz == 1 { idx += runeSz continue } runes[r]++ idx += runeSz } return } /* SquashConsec squashes/collapses consecutive instances of sequences seq to a single instance. It is expected that s is UTF-8 or a compatible subset (e.g. ASCII). If seq is nil/empty, any n+1 consecutive instance of *any rune* will be squashed to a single instance. This is *much* more performant (as it simply wraps [SquashConsecRunesAll]) at the cost of lack of scoping. If you are trying to squash/collapse whitespace instead, [SquashWhitespace] may be more apropos. If you want more fine-grained control over sequence replacement, see [SquashMap]. If s is an empty string, SquashConsec will return `s` as-is. */ func SquashConsec(s string, seq ...string) (squashed string) { var idx int var sIdx int var kLen int var start int var seqIdx int var nextNum int var nextPos int var sb *strings.Builder squashed = s if len(s) == 0 { return } if len(seq) == 0 { squashed = SquashConsecRunesAll(s) return } for seqIdx = range seq { if seq[seqIdx] == "" { continue } sb = new(strings.Builder) sb.Grow(len(squashed)) kLen = len(seq[seqIdx]) for idx = 0; idx < len(squashed); { sIdx = strings.Index(squashed[idx:], seq[seqIdx]) if sIdx < 0 { sb.WriteString(squashed[idx:]) break } start = idx + sIdx sb.WriteString(squashed[idx:start]) nextNum = 0 nextPos = start for strings.HasPrefix(squashed[nextPos:], seq[seqIdx]) { nextNum++ nextPos += kLen } if nextNum >= 1 { sb.WriteString(seq[seqIdx]) } idx = nextPos } squashed = sb.String() } return } /* SquashConsecRunes squashes/collapses consecutive instances of sequences seq with a single instance. It is expected that s is UTF-8 or a compatible subset (e.g. ASCII). If seq is nil/empty, any n+1 consecutive instance of *any rune* will be squashed to a single instance. This is *much* more performant (as it simply wraps [SquashConsecRunesAll]) at the cost of lack of scoping. If s is an empty string, SquashConsecRunes will return `s` as-is. */ func SquashConsecRunes(s string, seq ...rune) (squashed string) { var r rune var idx int var prev rune var seqIdx int var runeSz int var sb *strings.Builder squashed = s if len(s) == 0 { return } if len(seq) == 0 { squashed = SquashConsecRunesAll(s) return } for seqIdx = range seq { if seq[seqIdx] < 0 { continue } sb = new(strings.Builder) sb.Grow(len(squashed)) prev = -1 for idx = 0; idx < len(squashed); { r, runeSz = utf8.DecodeRuneInString(squashed[idx:]) if r == utf8.RuneError && runeSz == 1 { prev = -1 sb.WriteString(squashed[idx : idx+runeSz]) } else { if r != seq[seqIdx] || prev != r { prev = r sb.WriteRune(r) } } idx += runeSz } squashed = sb.String() } return } /* SquashConsecRunesAll squashes/condenses/collapses all consecutive occurrences of every rune in `s` to a single instance. It is expected that s is UTF-8 or a compatible subset (e.g. ASCII); non-UTF-8 runes will be written as-is to squashed (even if duplicate). That is to say, for an `s` of: * `fo` * `foo` * `fooo` * `foooo` SquashConsecRunesAll would return `fo` for all of them. SquashConsecRunesAll will be a no-op and return an empty string if `s` is an empty string. */ func SquashConsecRunesAll(s string) (squashed string) { var r rune var runeSz int var prev rune = -1 var sb *strings.Builder squashed = s if len(s) == 0 { return } sb = new(strings.Builder) sb.Grow(len(s)) for idx := 0; idx < len(squashed); { r, runeSz = utf8.DecodeRuneInString(squashed[idx:]) if r == utf8.RuneError && runeSz == 1 { prev = -1 sb.WriteString(squashed[idx : idx+runeSz]) } else { if prev != r { prev = r sb.WriteRune(r) } } idx += runeSz } squashed = sb.String() return } /* SquashMap offers replacement of consecutive string series. It is expected that s is UTF-8 or a compatible subset (e.g. ASCII). If single is set to true, *single/standalone* instances of a sequence in replaceMap keys will also be replaced with its corresponding value. Be aware that this function is not ideal for very large replaceMap maps or for very large strings, as s will be passed over for each key in replaceMap. If dealing with very large strings or a very large explicit set of replacements, you may want to directly use [strings.Map] or [strings.NewReplacer] (if multi-rune replacement is needed) with your own logic. Matching string replacements are done longest-to-shortest sequentially. This means that given the string `fooo` (and assuming single==true), and a replaceMap of: map[string]string{ "o": "a", "oo": "b", } the result will be "fba", not: * "faaa" * "fab" Thus you may want to execute several calls if this is undesired. Keys that are the same length are ordered alphabetically/alphanumerically so this function is deterministic. Empty keys in replaceMap (but NOT empty values) are skipped. Keys in replaceMap that are not UTF-8 are skipped. Note that this function offers high-granularity control over replacements at the cost of complex setup. If you want more simple squashing/collapsing of whitespace, [SquashWhitespace] may be more apropos. If you want more simple squashing/collapsing of non-whitespace, [SquashConsec] may be more apropos. If replaceMap is nil or empty, s will be returned as-is. If s is an empty string, SquashMap will no-op. */ func SquashMap(s string, replaceMap map[string]string, single bool) (squashed string) { var idx int var sIdx int var k string var v string var start int var nextNum int var nextPos int var sb *strings.Builder type kLen struct { // k is the Key name. k string // r is the replacement string. r string // l is the length of k (in UTF-8 runes). l int // bl is the length of k (in bytes). bl int } var kl kLen var keysByLen []kLen squashed = s if len(replaceMap) == 0 || len(s) == 0 { return } keysByLen = make([]kLen, 0, len(replaceMap)) for k, v = range replaceMap { if k == "" { continue } if !utf8.ValidString(k) { continue } keysByLen = append( keysByLen, kLen{ k: k, r: v, l: utf8.RuneCountInString(k), bl: len(k), }, ) } slices.SortFunc( keysByLen, func(a, b kLen) (cmp int) { cmp = 0 // Sort so that the longer comes first. if a.l > b.l { cmp = -1 } else if a.l < b.l { cmp = 1 } else if a.l == b.l { if a.k < b.k { cmp = -1 } else if a.k > b.k { cmp = 1 } } return }, ) for _, kl = range keysByLen { switch strings.Count(squashed, kl.k) { case 0: continue case 1: if !single { continue } squashed = strings.ReplaceAll(squashed, kl.k, kl.r) default: sb = new(strings.Builder) sb.Grow(len(squashed)) for idx = 0; idx < len(squashed); { if sIdx = strings.Index(squashed[idx:], kl.k); sIdx < 0 { sb.WriteString(squashed[idx:]) break } start = idx + sIdx sb.WriteString(squashed[idx:start]) nextNum = 0 nextPos = start for strings.HasPrefix(squashed[nextPos:], kl.k) { nextNum++ nextPos += kl.bl } if nextNum > 1 || (nextNum == 1 && single) { sb.WriteString(kl.r) } else { sb.WriteString(kl.k) } idx = nextPos } squashed = sb.String() } } return } /* SquashWhitespace is used to collapse/squash consecutive whitespace. It is expected that s is unicode or a compatible subset (e.g. ASCII). If single is set to true, *single/standalone* instances of whitespace will also be replaced with ws. ws is a string that should replace *consecutive* whitespace. ws does not necessarily have to be a whitespace string, and may be empty. If you are trying to squash/collapse non-whitespace, [SquashConsec] is more apropos. If you want more fine-grained control over sequence replacement, see [SquashMap]. */ func SquashWhitespace(s, ws string, single bool) (squashed string) { var r rune var runeSz int var wsStart int var invalid bool var prevWs uint64 var sb *strings.Builder squashed = s if s == "" { return } sb = new(strings.Builder) sb.Grow(len(s)) for rIdx := 0; rIdx < len(s); { r, runeSz = utf8.DecodeRuneInString(s[rIdx:]) invalid = r == utf8.RuneError && runeSz == 1 if invalid || !unicode.IsSpace(r) { if prevWs > 1 || (prevWs == 1 && single) { sb.WriteString(ws) } else if prevWs == 1 { sb.WriteString(s[wsStart:rIdx]) } prevWs = 0 if invalid { // Invalid UTF-8 rune; write byte as-is and advance index. sb.WriteString(s[rIdx : rIdx+runeSz]) } else { sb.WriteRune(r) } rIdx += runeSz continue } if prevWs == 0 { wsStart = rIdx } prevWs++ rIdx += runeSz } if prevWs > 1 || (prevWs == 1 && single) { sb.WriteString(ws) } else if prevWs == 1 { sb.WriteString(s[wsStart:]) } squashed = sb.String() return } /* StripWhitespace removes all leading, trailing, and INNER whitespace from unicode string `s`. This is done by mapping each rune in `s`. This may use far more allocations than necessary if `s` is mostly NON-whitespace; in that case, it is better to use [RemoveWhitespace]. */ func StripWhitespace(s string) (stripped string) { // https://stackoverflow.com/questions/32081808/strip-all-whitespace-from-a-string/32081891#32081891 if s == "" { return } stripped = strings.Map( func(c rune) rune { if unicode.IsSpace(c) { return -1 } return c }, s, ) return } /* TrimLines is like [strings.TrimSpace] but operates on *each line* of s. It is *NIX-newline (`\n`) vs. Windows-newline (`\r\n`) agnostic. The first encountered linebreak (`\n` vs. `\r\n`) are assumed to be the canonical linebreak for the rest of s. left, if true, performs a [TrimSpaceLeft] on each line (retaining the newline). right, if true, performs a [TrimSpaceRight] on each line (retaining the newline). */ func TrimLines(s string, left, right bool) (trimmed string) { var sl string var nl string var sb strings.Builder // These conditions functionally won't do anything, so just return the input as-is. if s == "" { return } if !left && !right { trimmed = s return } for line := range strings.Lines(s) { nl = getNewLine(line) sl = strings.TrimSuffix(line, nl) if left && right { sl = strings.TrimSpace(sl) } else if left { sl = TrimSpaceLeft(sl) } else if right { sl = TrimSpaceRight(sl) } sb.WriteString(sl) sb.WriteString(nl) } trimmed = sb.String() return } // TrimSpaceLeft is like [strings.TrimSpace] but only removes leading whitespace from string `s`. func TrimSpaceLeft(s string) (trimmed string) { trimmed = strings.TrimLeftFunc(s, unicode.IsSpace) return } /* TrimSpaceRight is like [strings.TrimSpace] but only removes trailing whitespace from string s. */ func TrimSpaceRight(s string) (trimmed string) { trimmed = strings.TrimRightFunc(s, unicode.IsSpace) return } // getAsciiCharMap returns a lookup "table" for ASCII characters. func getAsciiCharMap(allowCtl, allowPrint, allowExt, allowWs bool, incl, excl []byte) (charmap [256]bool) { var idx uint8 if allowCtl { for idx < 0x1f { charmap[idx] = true idx++ } } else { idx = 0x1f } if allowPrint { for idx < 0x7f { charmap[idx] = true idx++ } } else { idx = 0x7f } if allowExt { for { charmap[idx] = true if idx == 0xff { break } idx++ } } else { idx = 0xff } if allowWs { charmap['\t'] = true charmap['\n'] = true charmap['\r'] = true } if incl != nil && len(incl) > 0 { for _, idx = range incl { charmap[idx] = true } } if excl != nil && len(excl) > 0 { for _, idx = range excl { charmap[idx] = false } } return } // getNewLine is too unpredictable/nuanced to be used as part of a public API promise so it isn't exported. func getNewLine(s string) (nl string) { if strings.HasSuffix(s, "\r\n") { nl = "\r\n" } else if strings.HasSuffix(s, "\n") { nl = "\n" } return }