From 061fc3f82d44fa8da0a2981fd4afab3bfe6df485 Mon Sep 17 00:00:00 2001 From: linzhenqi Date: Sat, 14 Feb 2026 01:48:12 +0800 Subject: [PATCH] [Enhancement](pyudf) add helper command to show more py info --- be/src/service/backend_service.cpp | 42 +++++ be/src/service/backend_service.h | 5 + be/src/udf/python/python_env.cpp | 42 +++++ be/src/udf/python/python_env.h | 12 ++ .../org/apache/doris/nereids/DorisLexer.g4 | 3 + .../org/apache/doris/nereids/DorisParser.g4 | 5 + .../nereids/parser/LogicalPlanBuilder.java | 13 ++ .../doris/nereids/trees/plans/PlanType.java | 2 + .../commands/ShowPythonPackagesCommand.java | 176 ++++++++++++++++++ .../commands/ShowPythonVersionsCommand.java | 140 ++++++++++++++ .../trees/plans/visitor/CommandVisitor.java | 10 + .../apache/doris/common/GenericPoolTest.java | 12 ++ .../doris/utframe/MockedBackendFactory.java | 12 ++ gensrc/thrift/BackendService.thrift | 19 ++ .../test_show_python_packages_command.groovy | 55 ++++++ .../test_show_python_versions_command.groovy | 59 ++++++ 16 files changed, 607 insertions(+) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowPythonPackagesCommand.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowPythonVersionsCommand.java create mode 100644 regression-test/suites/nereids_p0/show/test_show_python_packages_command.groovy create mode 100644 regression-test/suites/nereids_p0/show/test_show_python_versions_command.groovy diff --git a/be/src/service/backend_service.cpp b/be/src/service/backend_service.cpp index dd71e2c92b218b..c1abffa62b5b52 100644 --- a/be/src/service/backend_service.cpp +++ b/be/src/service/backend_service.cpp @@ -71,6 +71,7 @@ #include "runtime/routine_load/routine_load_task_executor.h" #include "runtime/stream_load/stream_load_context.h" #include "runtime/stream_load/stream_load_recorder.h" +#include "udf/python/python_env.h" #include "util/arrow/row_batch.h" #include "util/defer_op.h" #include "util/runtime_profile.h" @@ -1311,5 +1312,46 @@ void BaseBackendService::test_storage_connectivity(TTestStorageConnectivityRespo response.__set_status(status.to_thrift()); } +void BaseBackendService::get_python_envs(std::vector& result) { + auto& manager = PythonVersionManager::instance(); + const auto& envs = manager.get_envs(); + + std::string env_type_str; + switch (manager.env_type()) { + case PythonEnvType::CONDA: + env_type_str = "conda"; + break; + case PythonEnvType::VENV: + env_type_str = "venv"; + break; + } + + for (const auto& env : envs) { + TPythonEnvInfo info; + info.__set_env_name(env.env_name); + info.__set_full_version(env.python_version.full_version); + info.__set_env_type(env_type_str); + info.__set_base_path(env.python_version.base_path); + info.__set_executable_path(env.python_version.executable_path); + result.push_back(std::move(info)); + } +} + +void BaseBackendService::get_python_packages(std::vector& result, + const std::string& python_version) { + PythonVersion version; + THROW_IF_ERROR(PythonVersionManager::instance().get_version(python_version, &version)); + + std::vector> packages; + THROW_IF_ERROR(list_installed_packages(version, &packages)); + + for (const auto& [name, ver] : packages) { + TPythonPackageInfo info; + info.__set_package_name(name); + info.__set_version(ver); + result.push_back(std::move(info)); + } +} + #include "common/compile_check_end.h" } // namespace doris diff --git a/be/src/service/backend_service.h b/be/src/service/backend_service.h index 6c43f69170d5a3..4796412ef5f9eb 100644 --- a/be/src/service/backend_service.h +++ b/be/src/service/backend_service.h @@ -122,6 +122,11 @@ class BaseBackendService : public BackendServiceIf { void test_storage_connectivity(TTestStorageConnectivityResponse& response, const TTestStorageConnectivityRequest& request) override; + void get_python_envs(std::vector& result) override; + + void get_python_packages(std::vector& result, + const std::string& python_version) override; + //////////////////////////////////////////////////////////////////////////// // begin cloud backend functions //////////////////////////////////////////////////////////////////////////// diff --git a/be/src/udf/python/python_env.cpp b/be/src/udf/python/python_env.cpp index 2d122a235f1d4b..9d50851d2cdff3 100644 --- a/be/src/udf/python/python_env.cpp +++ b/be/src/udf/python/python_env.cpp @@ -288,4 +288,46 @@ Status PythonVersionManager::init(PythonEnvType env_type, const fs::path& python return Status::OK(); } +Status list_installed_packages(const PythonVersion& version, + std::vector>* packages) { + DCHECK(packages != nullptr); + if (!version.is_valid()) { + return Status::InvalidArgument("Invalid python version: {}", version.to_string()); + } + + // Run pip list --format=json to get installed packages + std::string cmd = + fmt::format("\"{}\" -m pip list --format=json 2>/dev/null", version.executable_path); + FILE* pipe = popen(cmd.c_str(), "r"); + if (!pipe) { + return Status::InternalError("Failed to run pip list for python version: {}", + version.full_version); + } + + std::string result; + char buf[4096]; + while (fgets(buf, sizeof(buf), pipe)) { + result += buf; + } + int ret = pclose(pipe); + if (ret != 0) { + return Status::InternalError( + "pip list failed for python version: {}, exit code: {}, output: {}", + version.full_version, ret, result); + } + + // Parse JSON output: [{"name": "pkg", "version": "1.0"}, ...] + // Simple JSON parsing without external library + // Each entry looks like: {"name": "package_name", "version": "1.2.3"} + static std::regex entry_re( + R"REGEX(\{\s*"name"\s*:\s*"([^"]+)"\s*,\s*"version"\s*:\s*"([^"]+)"\s*\})REGEX"); + auto begin = std::sregex_iterator(result.begin(), result.end(), entry_re); + auto end = std::sregex_iterator(); + for (auto it = begin; it != end; ++it) { + packages->emplace_back((*it)[1].str(), (*it)[2].str()); + } + + return Status::OK(); +} + } // namespace doris diff --git a/be/src/udf/python/python_env.h b/be/src/udf/python/python_env.h index 4d3a5acca60407..7dcab05c46df1b 100644 --- a/be/src/udf/python/python_env.h +++ b/be/src/udf/python/python_env.h @@ -18,6 +18,7 @@ #pragma once #include +#include #include "common/status.h" @@ -90,6 +91,8 @@ class PythonEnvScanner { Status get_version(const std::string& runtime_version, PythonVersion* version) const; + const std::vector& get_envs() const { return _envs; } + std::string root_path() const { return _env_root_path.string(); } virtual PythonEnvType env_type() const = 0; @@ -146,12 +149,21 @@ class PythonVersionManager { return _env_scanner->get_version(runtime_version, version); } + const std::vector& get_envs() const { return _env_scanner->get_envs(); } + + PythonEnvType env_type() const { return _env_scanner->env_type(); } + std::string to_string() const { return _env_scanner->to_string(); } private: std::unique_ptr _env_scanner; }; +// List installed pip packages for a given Python version. +// Returns pairs of (package_name, version). +Status list_installed_packages(const PythonVersion& version, + std::vector>* packages); + } // namespace doris namespace std { diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 index 12a73beb2c5559..0b2b5bf1ef7c95 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 @@ -414,6 +414,7 @@ PARAMETER: 'PARAMETER'; PARSED: 'PARSED'; PARTITION: 'PARTITION'; PARTITIONS: 'PARTITIONS'; +PACKAGES: 'PACKAGES'; PASSWORD: 'PASSWORD'; PASSWORD_EXPIRE: 'PASSWORD_EXPIRE'; PASSWORD_HISTORY: 'PASSWORD_HISTORY'; @@ -431,6 +432,7 @@ PLAN: 'PLAN'; PLAY: 'PLAY'; PRIVILEGES: 'PRIVILEGES'; PROCESS: 'PROCESS'; +PYTHON: 'PYTHON'; PLUGIN: 'PLUGIN'; PLUGINS: 'PLUGINS'; POLICY: 'POLICY'; @@ -603,6 +605,7 @@ VAULT: 'VAULT'; VAULTS: 'VAULTS'; VERBOSE: 'VERBOSE'; VERSION: 'VERSION'; +VERSIONS: 'VERSIONS'; VIEW: 'VIEW'; VIEWS: 'VIEWS'; WARM: 'WARM'; diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index be2c75019006a4..3b8e2af9eeba7b 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -471,6 +471,8 @@ supportedShowStatement (FROM |IN) tableName=multipartIdentifier ((FROM | IN) database=multipartIdentifier)? #showIndex | SHOW WARM UP JOB wildWhere? #showWarmUpJob + | SHOW PYTHON VERSIONS #showPythonVersions + | SHOW PYTHON PACKAGES IN STRING_LITERAL #showPythonPackages ; supportedLoadStatement @@ -2177,6 +2179,7 @@ nonReserved | PASSWORD_LOCK_TIME | PASSWORD_REUSE | PARTITIONS + | PACKAGES | PATH | PAUSE | PERCENT @@ -2196,6 +2199,7 @@ nonReserved | PROFILE | PROPERTIES | PROPERTY + | PYTHON | QUANTILE_STATE | QUANTILE_UNION | QUARTER @@ -2301,6 +2305,7 @@ nonReserved | VAULTS | VERBOSE | VERSION + | VERSIONS | VIEW | VIEWS | WARM diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index 82314c719a0dbc..f1909a0839141b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -833,6 +833,8 @@ import org.apache.doris.nereids.trees.plans.commands.ShowPrivilegesCommand; import org.apache.doris.nereids.trees.plans.commands.ShowProcCommand; import org.apache.doris.nereids.trees.plans.commands.ShowProcessListCommand; +import org.apache.doris.nereids.trees.plans.commands.ShowPythonPackagesCommand; +import org.apache.doris.nereids.trees.plans.commands.ShowPythonVersionsCommand; import org.apache.doris.nereids.trees.plans.commands.ShowQueryProfileCommand; import org.apache.doris.nereids.trees.plans.commands.ShowQueryStatsCommand; import org.apache.doris.nereids.trees.plans.commands.ShowQueuedAnalyzeJobsCommand; @@ -6487,6 +6489,17 @@ public LogicalPlan visitShowTrash(ShowTrashContext ctx) { return new ShowTrashCommand(); } + @Override + public LogicalPlan visitShowPythonVersions(DorisParser.ShowPythonVersionsContext ctx) { + return new ShowPythonVersionsCommand(); + } + + @Override + public LogicalPlan visitShowPythonPackages(DorisParser.ShowPythonPackagesContext ctx) { + String version = stripQuotes(ctx.STRING_LITERAL().getText()); + return new ShowPythonPackagesCommand(version); + } + @Override public LogicalPlan visitAdminCleanTrash(DorisParser.AdminCleanTrashContext ctx) { if (ctx.ON() != null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java index 0c5a16fc3efa76..8dca16bb780e56 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java @@ -307,6 +307,8 @@ public enum PlanType { SHOW_PROC_COMMAND, SHOW_PLUGINS_COMMAND, SHOW_PRIVILEGES_COMMAND, + SHOW_PYTHON_PACKAGES_COMMAND, + SHOW_PYTHON_VERSIONS_COMMAND, SHOW_REPLICA_DISTRIBUTION_COMMAND, SHOW_REPLICA_STATUS_COMMAND, SHOW_REPOSITORIES_COMMAND, diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowPythonPackagesCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowPythonPackagesCommand.java new file mode 100644 index 00000000000000..bb6216d05b09c6 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowPythonPackagesCommand.java @@ -0,0 +1,176 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.commands; + +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.ScalarType; +import org.apache.doris.common.ClientPool; +import org.apache.doris.common.ErrorCode; +import org.apache.doris.common.ErrorReport; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.ShowResultSet; +import org.apache.doris.qe.ShowResultSetMetaData; +import org.apache.doris.qe.StmtExecutor; +import org.apache.doris.system.Backend; +import org.apache.doris.thrift.BackendService; +import org.apache.doris.thrift.TNetworkAddress; +import org.apache.doris.thrift.TPythonPackageInfo; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * SHOW PYTHON PACKAGES IN '<version>' command. + * Shows pip packages installed for the given Python version, collected from all alive BEs. + */ +public class ShowPythonPackagesCommand extends ShowCommand { + private static final Logger LOG = LogManager.getLogger(ShowPythonPackagesCommand.class); + + private static final String[] TITLE_NAMES = {"Package", "Version"}; + private static final String[] TITLE_NAMES_INCONSISTENT = {"Package", "Version", "Consistent"}; + + private final String pythonVersion; + + public ShowPythonPackagesCommand(String pythonVersion) { + super(PlanType.SHOW_PYTHON_PACKAGES_COMMAND); + this.pythonVersion = pythonVersion; + } + + public String getPythonVersion() { + return pythonVersion; + } + + @Override + public ShowResultSetMetaData getMetaData() { + return getMetaData(true); + } + + private ShowResultSetMetaData getMetaData(boolean consistent) { + ShowResultSetMetaData.Builder builder = ShowResultSetMetaData.builder(); + String[] titles = consistent ? TITLE_NAMES : TITLE_NAMES_INCONSISTENT; + for (String title : titles) { + builder.addColumn(new Column(title, ScalarType.createVarchar(256))); + } + return builder.build(); + } + + @Override + public R accept(PlanVisitor visitor, C context) { + return visitor.visitShowPythonPackagesCommand(this, context); + } + + @Override + public ShowResultSet doRun(ConnectContext ctx, StmtExecutor executor) throws Exception { + if (!Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ConnectContext.get(), + PrivPredicate.ADMIN)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN"); + } + + ImmutableMap backendsInfo = Env.getCurrentSystemInfo().getAllBackendsByAllCluster(); + + // Collect packages from each alive BE + List> allBePackages = new ArrayList<>(); + for (Backend backend : backendsInfo.values()) { + if (!backend.isAlive()) { + continue; + } + BackendService.Client client = null; + TNetworkAddress address = null; + boolean ok = false; + try { + address = new TNetworkAddress(backend.getHost(), backend.getBePort()); + client = ClientPool.backendPool.borrowObject(address); + List packages = client.getPythonPackages(pythonVersion); + ok = true; + + Map pkgMap = new HashMap<>(); + for (TPythonPackageInfo pkg : packages) { + pkgMap.put(pkg.getPackageName(), pkg.getVersion()); + } + allBePackages.add(pkgMap); + } catch (Exception e) { + LOG.warn("Failed to get python packages from backend[{}]", backend.getId(), e); + } finally { + if (ok) { + ClientPool.backendPool.returnObject(address, client); + } else { + ClientPool.backendPool.invalidateObject(address, client); + } + } + } + + if (allBePackages.isEmpty()) { + return new ShowResultSet(getMetaData(), Lists.newArrayList()); + } + + // Check consistency across BEs + boolean consistent = true; + Map referencePackages = allBePackages.get(0); + for (int i = 1; i < allBePackages.size(); i++) { + if (!referencePackages.equals(allBePackages.get(i))) { + consistent = false; + break; + } + } + + List> rows = Lists.newArrayList(); + if (consistent) { + for (Map.Entry entry : referencePackages.entrySet()) { + List row = new ArrayList<>(); + row.add(entry.getKey()); + row.add(entry.getValue()); + rows.add(row); + } + } else { + // Collect all unique package names + Map> packageVersionCounts = new HashMap<>(); + for (Map pkgMap : allBePackages) { + for (Map.Entry entry : pkgMap.entrySet()) { + packageVersionCounts.computeIfAbsent(entry.getKey(), k -> new HashMap<>()) + .merge(entry.getValue(), 1, Integer::sum); + } + } + for (Map.Entry> entry : packageVersionCounts.entrySet()) { + String pkgName = entry.getKey(); + Map versionCounts = entry.getValue(); + boolean pkgConsistent = versionCounts.size() == 1 + && versionCounts.values().iterator().next() == allBePackages.size(); + for (Map.Entry vc : versionCounts.entrySet()) { + List row = new ArrayList<>(); + row.add(pkgName); + row.add(vc.getKey()); + row.add(pkgConsistent ? "Yes" : "No"); + rows.add(row); + } + } + } + + return new ShowResultSet(getMetaData(consistent), rows); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowPythonVersionsCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowPythonVersionsCommand.java new file mode 100644 index 00000000000000..16fab957bcef4a --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowPythonVersionsCommand.java @@ -0,0 +1,140 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.commands; + +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.ScalarType; +import org.apache.doris.common.ClientPool; +import org.apache.doris.common.ErrorCode; +import org.apache.doris.common.ErrorReport; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.ShowResultSet; +import org.apache.doris.qe.ShowResultSetMetaData; +import org.apache.doris.qe.StmtExecutor; +import org.apache.doris.system.Backend; +import org.apache.doris.thrift.BackendService; +import org.apache.doris.thrift.TNetworkAddress; +import org.apache.doris.thrift.TPythonEnvInfo; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * SHOW PYTHON VERSIONS command. + * Shows Python versions available on all alive backends (intersection). + */ +public class ShowPythonVersionsCommand extends ShowCommand { + private static final Logger LOG = LogManager.getLogger(ShowPythonVersionsCommand.class); + + private static final String[] TITLE_NAMES = { + "Version", "EnvName", "EnvType", "BasePath", "ExecutablePath" + }; + + public ShowPythonVersionsCommand() { + super(PlanType.SHOW_PYTHON_VERSIONS_COMMAND); + } + + @Override + public ShowResultSetMetaData getMetaData() { + ShowResultSetMetaData.Builder builder = ShowResultSetMetaData.builder(); + for (String title : TITLE_NAMES) { + builder.addColumn(new Column(title, ScalarType.createVarchar(256))); + } + return builder.build(); + } + + @Override + public R accept(PlanVisitor visitor, C context) { + return visitor.visitShowPythonVersionsCommand(this, context); + } + + @Override + public ShowResultSet doRun(ConnectContext ctx, StmtExecutor executor) throws Exception { + if (!Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ConnectContext.get(), + PrivPredicate.ADMIN)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN"); + } + + ImmutableMap backendsInfo = Env.getCurrentSystemInfo().getAllBackendsByAllCluster(); + List> allBeEnvs = new ArrayList<>(); + Set commonVersions = null; + + for (Backend backend : backendsInfo.values()) { + if (!backend.isAlive()) { + continue; + } + BackendService.Client client = null; + TNetworkAddress address = null; + boolean ok = false; + try { + address = new TNetworkAddress(backend.getHost(), backend.getBePort()); + client = ClientPool.backendPool.borrowObject(address); + List envs = client.getPythonEnvs(); + ok = true; + + allBeEnvs.add(envs); + Set versions = new HashSet<>(); + for (TPythonEnvInfo env : envs) { + versions.add(env.getFullVersion()); + } + if (commonVersions == null) { + commonVersions = versions; + } else { + commonVersions.retainAll(versions); + } + } catch (Exception e) { + LOG.warn("Failed to get python envs from backend[{}]", backend.getId(), e); + } finally { + if (ok) { + ClientPool.backendPool.returnObject(address, client); + } else { + ClientPool.backendPool.invalidateObject(address, client); + } + } + } + + List> rows = Lists.newArrayList(); + if (commonVersions != null && !allBeEnvs.isEmpty()) { + // Use envs from the first BE as reference, filtered to common versions + for (TPythonEnvInfo env : allBeEnvs.get(0)) { + if (commonVersions.contains(env.getFullVersion())) { + List row = new ArrayList<>(); + row.add(env.getFullVersion()); + row.add(env.getEnvName()); + row.add(env.getEnvType()); + row.add(env.getBasePath()); + row.add(env.getExecutablePath()); + rows.add(row); + } + } + } + + return new ShowResultSet(getMetaData(), rows); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java index 743dd9dbbb5568..0f1625c629b297 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java @@ -231,6 +231,8 @@ import org.apache.doris.nereids.trees.plans.commands.ShowPrivilegesCommand; import org.apache.doris.nereids.trees.plans.commands.ShowProcCommand; import org.apache.doris.nereids.trees.plans.commands.ShowProcessListCommand; +import org.apache.doris.nereids.trees.plans.commands.ShowPythonPackagesCommand; +import org.apache.doris.nereids.trees.plans.commands.ShowPythonVersionsCommand; import org.apache.doris.nereids.trees.plans.commands.ShowQueryProfileCommand; import org.apache.doris.nereids.trees.plans.commands.ShowQueryStatsCommand; import org.apache.doris.nereids.trees.plans.commands.ShowQueuedAnalyzeJobsCommand; @@ -717,6 +719,14 @@ default R visitShowPluginsCommand(ShowPluginsCommand showPluginsCommand, C conte return visitCommand(showPluginsCommand, context); } + default R visitShowPythonVersionsCommand(ShowPythonVersionsCommand showPythonVersionsCommand, C context) { + return visitCommand(showPythonVersionsCommand, context); + } + + default R visitShowPythonPackagesCommand(ShowPythonPackagesCommand showPythonPackagesCommand, C context) { + return visitCommand(showPythonPackagesCommand, context); + } + default R visitShowTrashCommand(ShowTrashCommand showTrashCommand, C context) { return visitCommand(showTrashCommand, context); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/GenericPoolTest.java b/fe/fe-core/src/test/java/org/apache/doris/common/GenericPoolTest.java index e761f07a90e9bf..dd5e4984e71f95 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/common/GenericPoolTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/common/GenericPoolTest.java @@ -35,6 +35,8 @@ import org.apache.doris.thrift.TNetworkAddress; import org.apache.doris.thrift.TPublishTopicRequest; import org.apache.doris.thrift.TPublishTopicResult; +import org.apache.doris.thrift.TPythonEnvInfo; +import org.apache.doris.thrift.TPythonPackageInfo; import org.apache.doris.thrift.TQueryIngestBinlogRequest; import org.apache.doris.thrift.TQueryIngestBinlogResult; import org.apache.doris.thrift.TRoutineLoadTask; @@ -246,6 +248,16 @@ public org.apache.doris.thrift.TTestStorageConnectivityResponse testStorageConne org.apache.doris.thrift.TTestStorageConnectivityRequest request) throws TException { return null; } + + @Override + public List getPythonEnvs() throws TException { + return null; + } + + @Override + public List getPythonPackages(String pythonVersion) throws TException { + return null; + } } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/utframe/MockedBackendFactory.java b/fe/fe-core/src/test/java/org/apache/doris/utframe/MockedBackendFactory.java index 0d4a109921a1dd..5c20a2b64f6b90 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/utframe/MockedBackendFactory.java +++ b/fe/fe-core/src/test/java/org/apache/doris/utframe/MockedBackendFactory.java @@ -55,6 +55,8 @@ import org.apache.doris.thrift.TNetworkAddress; import org.apache.doris.thrift.TPublishTopicRequest; import org.apache.doris.thrift.TPublishTopicResult; +import org.apache.doris.thrift.TPythonEnvInfo; +import org.apache.doris.thrift.TPythonPackageInfo; import org.apache.doris.thrift.TQueryIngestBinlogRequest; import org.apache.doris.thrift.TQueryIngestBinlogResult; import org.apache.doris.thrift.TRoutineLoadTask; @@ -478,6 +480,16 @@ public org.apache.doris.thrift.TTestStorageConnectivityResponse testStorageConne response.setStatus(new TStatus(TStatusCode.OK)); return response; } + + @Override + public java.util.List getPythonEnvs() throws TException { + return java.util.Collections.emptyList(); + } + + @Override + public java.util.List getPythonPackages(String pythonVersion) throws TException { + return java.util.Collections.emptyList(); + } } // The default Brpc service. diff --git a/gensrc/thrift/BackendService.thrift b/gensrc/thrift/BackendService.thrift index 684f2404fcbe20..44c1e7cd94c619 100644 --- a/gensrc/thrift/BackendService.thrift +++ b/gensrc/thrift/BackendService.thrift @@ -378,6 +378,19 @@ struct TTestStorageConnectivityResponse { 1: optional Status.TStatus status; } +struct TPythonEnvInfo { + 1: optional string env_name // e.g. "myenv" + 2: optional string full_version // e.g. "3.9.16" + 3: optional string env_type // "conda" or "venv" + 4: optional string base_path // e.g. "/opt/miniconda3/envs/myenv" + 5: optional string executable_path +} + +struct TPythonPackageInfo { + 1: optional string package_name + 2: optional string version +} + service BackendService { AgentService.TAgentResult submit_tasks(1:list tasks); @@ -431,4 +444,10 @@ service BackendService { // Test storage connectivity (S3, HDFS, etc.) TTestStorageConnectivityResponse test_storage_connectivity(1:TTestStorageConnectivityRequest request); + + // Get Python environments available on this BE + list get_python_envs(); + + // Get installed pip packages for a specific Python version + list get_python_packages(1:string python_version); } diff --git a/regression-test/suites/nereids_p0/show/test_show_python_packages_command.groovy b/regression-test/suites/nereids_p0/show/test_show_python_packages_command.groovy new file mode 100644 index 00000000000000..0dbbc2ca958613 --- /dev/null +++ b/regression-test/suites/nereids_p0/show/test_show_python_packages_command.groovy @@ -0,0 +1,55 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite('test_show_python_packages_command') { + // get available Python versions so we can pick one for the packages test + def versions = sql 'SHOW PYTHON VERSIONS' + if (versions.size() == 0) { + return + } + def testVersion = versions[0][0] + + // Execute SHOW PYTHON PACKAGES IN '' + def result = sql "SHOW PYTHON PACKAGES IN '${testVersion}'" + + // There should be at least some packages installed + assertTrue(result.size() > 0, + "Expected at least some packages for Python ${testVersion}") + + // Collect all package names for later assertions + def packageNames = [] as Set + for (row in result) { + // Package name (column 0) should be non-empty + def pkgName = row[0] + assertNotNull(pkgName) + assertTrue(pkgName.length() > 0, 'Package name should not be empty') + + // Package version (column 1) should be non-empty + def pkgVersion = row[1] + assertNotNull(pkgVersion) + assertTrue(pkgVersion.length() > 0, + "Package version for '${pkgName}' should not be empty") + + packageNames.add(pkgName.toLowerCase()) + } + + // Verify that essential packages (pyarrow and pandas) are present + assertTrue(packageNames.contains('pyarrow'), + "pyarrow should be installed, found packages: ${packageNames}") + assertTrue(packageNames.contains('pandas'), + "pandas should be installed, found packages: ${packageNames}") +} diff --git a/regression-test/suites/nereids_p0/show/test_show_python_versions_command.groovy b/regression-test/suites/nereids_p0/show/test_show_python_versions_command.groovy new file mode 100644 index 00000000000000..dc322ea6b6b3dd --- /dev/null +++ b/regression-test/suites/nereids_p0/show/test_show_python_versions_command.groovy @@ -0,0 +1,59 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite('test_show_python_versions_command') { + def result = sql 'SHOW PYTHON VERSIONS' + + if (result.size() > 0) { + // Verify column structure: [Version, EnvName, EnvType, BasePath, ExecutablePath] + for (row in result) { + // Version (column 0) should be a non-empty version string like "3.9.16" + def version = row[0] + assertNotNull(version) + assertTrue(version.length() > 0, 'Version should not be empty') + assertTrue(version ==~ /\d+\.\d+(\.\d+)?/, + "Version '${version}' should match pattern x.y or x.y.z") + + // EnvName (column 1) should be non-empty + def envName = row[1] + assertNotNull(envName) + assertTrue(envName.length() > 0, 'EnvName should not be empty') + + // EnvType (column 2) must be either "conda" or "venv" + def envType = row[2] + assertTrue(envType == 'conda' || envType == 'venv', + "EnvType '${envType}' should be 'conda' or 'venv'") + + // BasePath (column 3) should be a non-empty path + def basePath = row[3] + assertNotNull(basePath) + assertTrue(basePath.length() > 0, 'BasePath should not be empty') + assertTrue(basePath.startsWith('/'), + "BasePath '${basePath}' should be an absolute path") + + // ExecutablePath (column 4) should be a non-empty path + def execPath = row[4] + assertNotNull(execPath) + assertTrue(execPath.length() > 0, 'ExecutablePath should not be empty') + assertTrue(execPath.startsWith('/'), + "ExecutablePath '${execPath}' should be an absolute path") + } + + // Verify uniqueness of versions in the result + result.collect { it[0] } as Set + } +}