forgejo/build/lint-locale-usage/lint-locale-usage.go

// Copyright 2023 The Gitea Authors. All rights reserved.
// Copyright 2025 The Forgejo Authors. All rights reserved.
// SPDX-License-Identifier: MIT

package main

import (
	"bufio"
	"errors"
	"flag"
	"fmt"
	"go/token"
	"io/fs"
	"os"
	"path/filepath"
	"sort"
	"strings"

	"forgejo.org/modules/container"
	"forgejo.org/modules/translation/localeiter"
)

// this works by first gathering all valid source string IDs from `en-US` reference files
// and then checking if all used source strings are actually defined

type LocatedError struct {
	Location string
	Kind     string
	Err      error
}

func (e LocatedError) Error() string {
	var sb strings.Builder

	sb.WriteString(e.Location)
	sb.WriteString(":\t")
	if e.Kind != "" {
		sb.WriteString(e.Kind)
		sb.WriteString(": ")
	}
	sb.WriteString("ERROR: ")
	sb.WriteString(e.Err.Error())

	return sb.String()
}

func InitLocaleTrFunctions() map[string][]uint {
	ret := make(map[string][]uint)

	f0 := []uint{0}
	ret["Tr"] = f0
	ret["TrString"] = f0
	ret["TrHTML"] = f0

	ret["TrPluralString"] = []uint{1}
	ret["TrN"] = []uint{1, 2}

	return ret
}

type Handler struct {
	OnMsgid            func(fset *token.FileSet, pos token.Pos, msgid string)
	OnMsgidPrefix      func(fset *token.FileSet, pos token.Pos, msgidPrefix string, truncated bool)
	OnUnexpectedInvoke func(fset *token.FileSet, pos token.Pos, funcname string, argc int)
	OnWarning          func(fset *token.FileSet, pos token.Pos, msg string)
	LocaleTrFunctions  map[string][]uint
}

type StringTrie interface {
	Matches(key []string) bool
}

type StringTrieMap map[string]StringTrie

func (m StringTrieMap) Matches(key []string) bool {
	if len(key) == 0 || m == nil {
		return true
	}
	value, ok := m[key[0]]
	if !ok {
		return false
	}
	if value == nil {
		return true
	}
	return value.Matches(key[1:])
}

func (m StringTrieMap) Insert(key []string) {
	if m == nil {
		return
	}

	switch len(key) {
	case 0:
		return

	case 1:
		m[key[0]] = nil

	default:
		if value, ok := m[key[0]]; ok {
			if value == nil {
				return
			}
		} else {
			m[key[0]] = make(StringTrieMap)
		}
		m[key[0]].(StringTrieMap).Insert(key[1:])
	}
}

func ParseAllowedMaskedUsages(fname string, usedMsgids container.Set[string], allowedMaskedPrefixes StringTrieMap, chkMsgid func(msgid string) bool) error {
	file, err := os.Open(fname)
	if err != nil {
		return LocatedError{
			Location: fname,
			Kind:     "Open",
			Err:      err,
		}
	}
	defer file.Close()

	scanner := bufio.NewScanner(file)
	lno := 0
	for scanner.Scan() {
		lno++
		line := strings.TrimSpace(scanner.Text())
		if line == "" || strings.HasPrefix(line, "#") {
			continue
		}
		if linePrefix, found := strings.CutSuffix(line, "."); found {
			allowedMaskedPrefixes.Insert(strings.Split(linePrefix, "."))
		} else {
			if !chkMsgid(line) {
				return LocatedError{
					Location: fmt.Sprintf("%s: line %d", fname, lno),
					Kind:     "undefined msgid",
					Err:      errors.New(line),
				}
			}
			usedMsgids.Add(line)
		}
	}
	if err := scanner.Err(); err != nil {
		return LocatedError{
			Location: fname,
			Kind:     "Scanner",
			Err:      err,
		}
	}
	return nil
}

// Truncating a message id prefix to the last dot
func PrepareMsgidPrefix(s string) (string, bool) {
	index := strings.LastIndexByte(s, 0x2e)
	if index == -1 {
		return "", true
	}
	return s[:index], index != len(s)-1
}

func Usage() {
	outp := flag.CommandLine.Output()
	fmt.Fprintf(outp, "Usage of %s:\n", os.Args[0])
	flag.PrintDefaults()

	fmt.Fprintf(outp, "\nThis command assumes that it gets started from the project root directory.\n")

	fmt.Fprintf(outp, "\nExit codes:\n")
	for _, i := range []string{
		"0\tsuccess, no issues found",
		"1\tunable to walk directory tree",
		"2\tunable to parse locale ini/json files",
		"3\tunable to parse go or text/template files",
		"4\tfound missing message IDs",
		"5\tfound unused message IDs",
	} {
		fmt.Fprintf(outp, "\t%s\n", i)
	}

	fmt.Fprintf(outp, "\nSpecial Go doc comments:\n")
	for _, i := range []string{
		"//llu:returnsTrKey",
		"\tcan be used in front of functions to indicate",
		"\tthat the function returns message IDs",
		"\tWARNING: this currently doesn't support nested functions properly",
		"",
		"//llu:returnsTrKeySuffix prefix.",
		"\tsimilar to llu:returnsTrKey, but the given prefix is prepended",
		"\tto the found strings before interpreting them as msgids",
		"",
		"// llu:TrKeys",
		"\tcan be used in front of 'const' and 'var' blocks",
		"\tin order to mark all contained strings as message IDs",
		"",
		"// llu:TrKeysSuffix prefix.",
		"\tlike llu:returnsTrKeySuffix, but for 'const' and 'var' blocks",
	} {
		if i == "" {
			fmt.Fprintf(outp, "\n")
		} else {
			fmt.Fprintf(outp, "\t%s\n", i)
		}
	}
}

//nolint:forbidigo
func main() {
	allowMissingMsgids := false
	allowUnusedMsgids := false
	usedMsgids := make(container.Set[string])
	allowedMaskedPrefixes := make(StringTrieMap)

	// It's possible for execl to hand us an empty os.Args.
	if len(os.Args) == 0 {
		flag.CommandLine = flag.NewFlagSet("lint-locale-usage", flag.ExitOnError)
	} else {
		flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError)
	}
	flag.CommandLine.Usage = Usage
	flag.Usage = Usage

	flag.BoolVar(
		&allowMissingMsgids,
		"allow-missing-msgids",
		false,
		"don't return an error code if missing message IDs are found",
	)
	flag.BoolVar(
		&allowUnusedMsgids,
		"allow-unused-msgids",
		false,
		"don't return an error code if unused message IDs are found",
	)

	msgids := make(container.Set[string])

	localeFile := filepath.Join(filepath.Join("options", "locale"), "locale_en-US.ini")
	localeContent, err := os.ReadFile(localeFile)
	if err != nil {
		fmt.Printf("%s:\tERROR: %s\n", localeFile, err.Error())
		os.Exit(2)
	}

	if err = localeiter.IterateMessagesContent(localeContent, func(trKey, trValue string) error {
		msgids[trKey] = struct{}{}
		return nil
	}); err != nil {
		fmt.Printf("%s:\tERROR: %s\n", localeFile, err.Error())
		os.Exit(2)
	}

	localeFile = filepath.Join(filepath.Join("options", "locale_next"), "locale_en-US.json")
	localeContent, err = os.ReadFile(localeFile)
	if err != nil {
		fmt.Printf("%s:\tERROR: %s\n", localeFile, err.Error())
		os.Exit(2)
	}

	if err := localeiter.IterateMessagesNextContent(localeContent, func(trKey, pluralForm, trValue string) error {
		// ignore plural form
		msgids[trKey] = struct{}{}
		return nil
	}); err != nil {
		fmt.Printf("%s:\tERROR: %s\n", localeFile, err.Error())
		os.Exit(2)
	}

	gotAnyMsgidError := false

	flag.Func(
		"allow-masked-usages-from",
		"supply a file containing a newline-separated list of allowed masked usages",
		func(argval string) error {
			return ParseAllowedMaskedUsages(argval, usedMsgids, allowedMaskedPrefixes, func(msgid string) bool {
				return msgids.Contains(msgid)
			})
		},
	)
	flag.Parse()

	onError := func(err error) {
		if err == nil {
			return
		}
		fmt.Println(err.Error())
		os.Exit(3)
	}

	handler := Handler{
		OnMsgidPrefix: func(fset *token.FileSet, pos token.Pos, msgidPrefix string, truncated bool) {
			msgidPrefixSplit := strings.Split(msgidPrefix, ".")
			if !truncated {
				allowedMaskedPrefixes.Insert(msgidPrefixSplit)
			} else if !allowedMaskedPrefixes.Matches(msgidPrefixSplit) {
				gotAnyMsgidError = true
				fmt.Printf("%s:\tmissing msgid prefix: %s\n", fset.Position(pos).String(), msgidPrefix)
			}
		},
		OnMsgid: func(fset *token.FileSet, pos token.Pos, msgid string) {
			if !msgids.Contains(msgid) {
				gotAnyMsgidError = true
				fmt.Printf("%s:\tmissing msgid: %s\n", fset.Position(pos).String(), msgid)
			} else {
				usedMsgids.Add(msgid)
			}
		},
		OnUnexpectedInvoke: func(fset *token.FileSet, pos token.Pos, funcname string, argc int) {
			gotAnyMsgidError = true
			fmt.Printf("%s:\tunexpected invocation of %s with %d arguments\n", fset.Position(pos).String(), funcname, argc)
		},
		OnWarning: func(fset *token.FileSet, pos token.Pos, msg string) {
			fmt.Printf("%s:\tWARNING: %s\n", fset.Position(pos).String(), msg)
		},
		LocaleTrFunctions: InitLocaleTrFunctions(),
	}

	if err := filepath.WalkDir(".", func(fpath string, d fs.DirEntry, err error) error {
		if err != nil {
			if os.IsNotExist(err) {
				return nil
			}
			return err
		}
		name := d.Name()
		if d.IsDir() {
			if name == "docker" || name == ".git" || name == "node_modules" {
				return fs.SkipDir
			}
		} else if name == "bindata.go" || fpath == "modules/translation/i18n/i18n_test.go" {
			// skip false positives
		} else if strings.HasSuffix(name, ".go") {
			onError(handler.HandleGoFile(fpath, nil))
		} else if strings.HasSuffix(name, ".tmpl") {
			if strings.HasPrefix(fpath, "tests") && strings.HasSuffix(name, ".ini.tmpl") {
				// skip false positives
			} else {
				onError(handler.HandleTemplateFile(fpath, nil))
			}
		}
		return nil
	}); err != nil {
		fmt.Printf("walkdir ERROR: %s\n", err.Error())
		os.Exit(1)
	}

	unusedMsgids := []string{}

	for msgid := range msgids {
		if !usedMsgids.Contains(msgid) && !allowedMaskedPrefixes.Matches(strings.Split(msgid, ".")) {
			unusedMsgids = append(unusedMsgids, msgid)
		}
	}

	sort.Strings(unusedMsgids)

	if len(unusedMsgids) != 0 {
		fmt.Printf("=== unused msgids (%d): ===\n", len(unusedMsgids))
		for _, msgid := range unusedMsgids {
			fmt.Printf("- %s\n", msgid)
		}
	}

	if !allowMissingMsgids && gotAnyMsgidError {
		os.Exit(4)
	}
	if !allowUnusedMsgids && len(unusedMsgids) != 0 {
		os.Exit(5)
	}
}