Skip to content

Commit 0d045cb

Browse files
committed
fix(language_support): batch package queries per call
1. Replace the per-package dpkg-query and apt-cache processes with one dpkg status snapshot and one apt-cache policy batch per public call, because apt-cache loads the whole package cache and applies the pins on every start, which took seconds per candidate package. 2. Keep the snapshot local to the call, so an installation or an apt update is visible to the next query instead of returning stale results. 3. Mark name:arch entries as their plain name too and accept only the installed status, so a package installed for one architecture is no longer reported as missing. 4. Force LC_ALL=C for apt-cache, otherwise a translated apt prints localized field names and no candidate version can be parsed. 5. Bound each command with a context timeout, so a stalled tool cannot block the query forever. 6. Give each tool its own command function with a literal binary path, so the program to run is never taken from a variable. 7. Add tests covering the batching, includeInstalled, the GNOME and dictionary exclusions, query failures and the snapshot refresh. Log: One language package query now runs a single dpkg-query and apt-cache. Influence: Language package queries no longer time out. fix(language_support): 语言包查询改为按调用批量执行 1. 不再逐包启动 dpkg-query 和 apt-cache,改为每次公开调用各执行一次 dpkg 状态读取和一次 apt-cache policy 批量查询;apt-cache 每次启动都要加载整个包缓存并计算 pin,单个候选包就要数秒。 2. 快照只在本次调用内有效,安装或 apt 更新后下一次查询即可看到新状态,不会返回过期结果。 3. name:arch 条目同时标记其原始包名,且只接受 installed 状态,某一架构已安装的包不再被判定为缺失。 4. apt-cache 强制使用 LC_ALL=C,否则翻译过的 apt 会输出本地化字段名,解析不到候选版本。 5. 每次调用加 context 超时,避免工具卡住时查询被无限期阻塞。 6. 两个工具各自使用独立的命令函数并写死二进制绝对路径,执行哪个程序不取自变量。 7. 补充测试,覆盖批量查询、includeInstalled、GNOME 与词典排除规则、查询失败以及快照刷新。 Log: 一次语言包查询现在只执行一次 dpkg-query 和一次 apt-cache。 PMS: BUG-379697 PMS: BUG-379687 PMS: BUG-379685 Influence: 语言包查询不再超时。
1 parent b7f122c commit 0d045cb

2 files changed

Lines changed: 391 additions & 79 deletions

File tree

‎language_support/lang_support.go‎

Lines changed: 123 additions & 78 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,17 @@
1-
// SPDX-FileCopyrightText: 2022 UnionTech Software Technology Co., Ltd.
1+
// SPDX-FileCopyrightText: 2022 - 2026 UnionTech Software Technology Co., Ltd.
22
//
33
// SPDX-License-Identifier: GPL-3.0-or-later
44

55
package language_support
66

77
import (
8-
"bytes"
8+
"context"
99
"encoding/csv"
1010
"io"
1111
"os"
1212
"os/exec"
1313
"strings"
14+
"time"
1415

1516
libLocale "github.com/linuxdeepin/go-lib/locale"
1617
)
@@ -35,84 +36,136 @@ func NewLanguageSupport() (ls *LanguageSupport, err error) {
3536
func (ls *LanguageSupport) Destroy() {
3637
}
3738

38-
func (ls *LanguageSupport) isPkgInstalled(name string) (bool, error) {
39-
output, err := exec.Command("/usr/bin/dpkg-query", "-W", "-f",
40-
"${db:Status-Status}", "--", name).Output()
41-
if err != nil {
42-
return false, err
43-
}
44-
return bytes.Equal(bytes.TrimSpace(output), []byte("installed")), nil
39+
// commandTimeout bounds a single dpkg-query or apt-cache run.
40+
const commandTimeout = 30 * time.Second
41+
42+
// dpkgQueryCommand and aptCacheCommand are variables so that tests can supply
43+
// canned output.
44+
var dpkgQueryCommand = func(args ...string) ([]byte, error) {
45+
ctx, cancel := context.WithTimeout(context.Background(), commandTimeout)
46+
defer cancel()
47+
return exec.CommandContext(ctx, "/usr/bin/dpkg-query", args...).Output()
4548
}
4649

47-
func (ls *LanguageSupport) isPkgInstallable(name string) (bool, error) {
48-
cmd := exec.Command("/usr/bin/apt-cache", "policy",
49-
"--", name)
50-
cmd.Env = []string{"LANG=en_US.UTF-8"}
51-
output, err := cmd.Output()
52-
if err != nil {
53-
return false, err
50+
var aptCacheCommand = func(args ...string) ([]byte, error) {
51+
ctx, cancel := context.WithTimeout(context.Background(), commandTimeout)
52+
defer cancel()
53+
cmd := exec.CommandContext(ctx, "/usr/bin/apt-cache", args...)
54+
// Only LC_ALL=C: a translated apt prints localized field names.
55+
cmd.Env = []string{"LC_ALL=C"}
56+
return cmd.Output()
57+
}
58+
59+
// pkgQuery is the dpkg/apt snapshot of a single call: one dpkg-query and one
60+
// apt-cache process, not cached across calls.
61+
type pkgQuery struct {
62+
installed map[string]bool
63+
candidates map[string]bool
64+
}
65+
66+
func newPkgQuery() *pkgQuery {
67+
return &pkgQuery{candidates: make(map[string]bool)}
68+
}
69+
70+
// isInstalled reads the whole dpkg database once; a multi-arch entry
71+
// ("name:arch") also marks the plain name.
72+
func (q *pkgQuery) isInstalled(name string) bool {
73+
if q.installed == nil {
74+
q.installed = make(map[string]bool)
75+
output, err := dpkgQueryCommand("-W", "-f",
76+
"${binary:Package} ${db:Status-Status}\n")
77+
if err == nil {
78+
for _, line := range strings.Split(string(output), "\n") {
79+
pkg, status, found := strings.Cut(line, " ")
80+
if !found || status != "installed" {
81+
continue
82+
}
83+
q.installed[pkg] = true
84+
if i := strings.IndexByte(pkg, ':'); i > 0 {
85+
q.installed[pkg[:i]] = true
86+
}
87+
}
88+
}
5489
}
90+
return q.installed[name]
91+
}
5592

56-
const candidate = "Candidate: "
57-
idx := bytes.Index(output, []byte(candidate))
58-
if idx == -1 {
59-
return false, nil
93+
// loadCandidates asks apt once for all given names; unknown names and
94+
// "(none)" candidates stay unmarked.
95+
func (q *pkgQuery) loadCandidates(names []string) {
96+
if len(names) == 0 {
97+
return
98+
}
99+
args := append([]string{"policy", "--"}, names...)
100+
output, err := aptCacheCommand(args...)
101+
if err != nil {
102+
return
60103
}
61104

62-
if bytes.HasPrefix(output[idx+len(candidate):], []byte("(none)")) {
63-
return false, nil
105+
var pkg string
106+
for _, line := range strings.Split(string(output), "\n") {
107+
if line == "" {
108+
continue
109+
}
110+
if line[0] != ' ' && strings.HasSuffix(line, ":") { // "package:" header
111+
pkg = strings.TrimSuffix(line, ":")
112+
continue
113+
}
114+
version, found := strings.CutPrefix(strings.TrimSpace(line), "Candidate:")
115+
if !found || pkg == "" {
116+
continue
117+
}
118+
version = strings.TrimSpace(version)
119+
q.candidates[pkg] = version != "" && version != "(none)"
64120
}
65-
return true, nil
66121
}
67122

68-
// ByPackageAndLocale get language support packages for a package and locale.
69-
func (ls *LanguageSupport) ByPackageAndLocale(
70-
package0 string, locale string, includeInstalled bool) (packages []string) {
71-
72-
packagesTemp := make(map[string]struct{})
123+
// candidateNames lists the packages offered for one trigger and locale.
124+
func (ls *LanguageSupport) candidateNames(package0, locale string) (names []string) {
73125
depMap := ls.pkgDepends[package0]
74126

75127
// check explicit entries for that locale
76128
for _, pkgs := range depMap[langCodeFromLocale(locale)] {
77-
for _, pkg := range pkgs {
78-
installable, err := ls.isPkgInstallable(pkg)
79-
if err != nil {
80-
continue
81-
}
82-
if installable {
83-
packagesTemp[pkg] = struct{}{}
84-
}
85-
}
129+
names = append(names, pkgs...)
86130
}
87131

88132
// check patterns for empty locale string (i. e. applies to any locale)
89133
for _, patternList := range depMap[""] {
90134
for _, pattern := range patternList {
91-
pkgs := expendPkgPattern(pattern, locale)
92-
for _, pkg := range pkgs {
93-
installable, err := ls.isPkgInstallable(pkg)
94-
if err != nil {
95-
continue
96-
}
97-
if installable {
98-
packagesTemp[pkg] = struct{}{}
99-
}
100-
}
135+
names = append(names, expendPkgPattern(pattern, locale)...)
101136
}
102137
}
138+
return
139+
}
103140

104-
if !includeInstalled { // not show installed
105-
for pkg := range packagesTemp {
106-
isInstalled, err := ls.isPkgInstalled(pkg)
107-
if err != nil {
141+
// packagesFor batches the candidates of all given triggers and applies the
142+
// filtering rules to the result.
143+
func (ls *LanguageSupport) packagesFor(
144+
triggers []string, locale string, includeInstalled bool, q *pkgQuery) []string {
145+
146+
names := make([]string, 0)
147+
seen := make(map[string]struct{})
148+
for _, trigger := range triggers {
149+
for _, name := range ls.candidateNames(trigger, locale) {
150+
if _, dup := seen[name]; dup {
108151
continue
109152
}
153+
seen[name] = struct{}{}
154+
names = append(names, name)
155+
}
156+
}
110157

111-
// filter out installed packages
112-
if isInstalled {
113-
delete(packagesTemp, pkg)
114-
}
158+
q.loadCandidates(names)
159+
160+
packagesTemp := make(map[string]struct{})
161+
for _, name := range names {
162+
if !q.candidates[name] {
163+
continue
115164
}
165+
if !includeInstalled && q.isInstalled(name) { // not show installed
166+
continue
167+
}
168+
packagesTemp[name] = struct{}{}
116169
}
117170

118171
// exclude Fcitx packages if GNOME desktop
@@ -142,40 +195,32 @@ func (ls *LanguageSupport) ByPackageAndLocale(
142195
// https://launchpad.net/bugs/1578821
143196
delete(packagesTemp, "hunspell-gl")
144197

145-
packages = make([]string, 0, len(packagesTemp))
198+
packages := make([]string, 0, len(packagesTemp))
146199
for pkg := range packagesTemp {
147200
packages = append(packages, pkg)
148201
}
149-
return
202+
return packages
203+
}
204+
205+
// ByPackageAndLocale get language support packages for a package and locale.
206+
func (ls *LanguageSupport) ByPackageAndLocale(
207+
package0 string, locale string, includeInstalled bool) (packages []string) {
208+
209+
return ls.packagesFor([]string{package0}, locale, includeInstalled, newPkgQuery())
150210
}
151211

152212
// ByLocale get language support packages for a locale
153213
func (ls *LanguageSupport) ByLocale(locale string, includeInstalled bool) []string {
154-
packagesTemp := make(map[string]struct{})
155-
for trigger := range ls.pkgDepends {
156-
var add bool
157-
if trigger == "" {
158-
add = true
159-
} else {
160-
pkgInstalled, err := ls.isPkgInstalled(trigger)
161-
if err == nil && pkgInstalled {
162-
add = true
163-
}
164-
}
214+
q := newPkgQuery()
165215

166-
if add {
167-
pkgs := ls.ByPackageAndLocale(trigger, locale, includeInstalled)
168-
for _, pkg := range pkgs {
169-
packagesTemp[pkg] = struct{}{}
170-
}
216+
var triggers []string
217+
for trigger := range ls.pkgDepends {
218+
if trigger == "" || q.isInstalled(trigger) {
219+
triggers = append(triggers, trigger)
171220
}
172221
}
173222

174-
packages := make([]string, 0, len(packagesTemp))
175-
for pkg := range packagesTemp {
176-
packages = append(packages, pkg)
177-
}
178-
return packages
223+
return ls.packagesFor(triggers, locale, includeInstalled, q)
179224
}
180225

181226
func expendPkgPattern(pattern, locale string) (pkgs []string) {

0 commit comments

Comments
 (0)