From mboxrd@z Thu Jan 1 00:00:00 1970 Return-Path: Received: from firstgate.proxmox.com (firstgate.proxmox.com [212.224.123.68]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by lists.proxmox.com (Postfix) with ESMTPS id A182A69685 for ; Mon, 13 Sep 2021 16:19:06 +0200 (CEST) Received: from firstgate.proxmox.com (localhost [127.0.0.1]) by firstgate.proxmox.com (Proxmox) with ESMTP id 952532AF44 for ; Mon, 13 Sep 2021 16:18:36 +0200 (CEST) Received: from proxmox-new.maurer-it.com (proxmox-new.maurer-it.com [94.136.29.106]) (using TLSv1.3 with cipher TLS_AES_256_GCM_SHA384 (256/256 bits) key-exchange X25519 server-signature RSA-PSS (2048 bits)) (No client certificate requested) by firstgate.proxmox.com (Proxmox) with ESMTPS id 73ED52AEA6 for ; Mon, 13 Sep 2021 16:18:32 +0200 (CEST) Received: from proxmox-new.maurer-it.com (localhost.localdomain [127.0.0.1]) by proxmox-new.maurer-it.com (Proxmox) with ESMTP id 4BF9C447F8 for ; Mon, 13 Sep 2021 16:18:32 +0200 (CEST) From: Dominik Csapak To: pbs-devel@lists.proxmox.com Date: Mon, 13 Sep 2021 16:18:24 +0200 Message-Id: <20210913141829.2171301-4-d.csapak@proxmox.com> X-Mailer: git-send-email 2.30.2 In-Reply-To: <20210913141829.2171301-1-d.csapak@proxmox.com> References: <20210913141829.2171301-1-d.csapak@proxmox.com> MIME-Version: 1.0 Content-Transfer-Encoding: 8bit X-SPAM-LEVEL: Spam detection results: 0 AWL -0.012 Adjusted score from AWL reputation of From: address BAYES_00 -1.9 Bayes spam probability is 0 to 1% KAM_ASCII_DIVIDERS 0.8 Spam that uses ascii formatting tricks KAM_DMARC_STATUS 0.01 Test Rule for DKIM or SPF Failure with Strict Alignment SPF_HELO_NONE 0.001 SPF: HELO does not publish an SPF Record SPF_PASS -0.001 SPF: sender matches SPF record Subject: [pbs-devel] [PATCH proxmox-backup v2 2/7] add 'pbs-shell' utility X-BeenThere: pbs-devel@lists.proxmox.com X-Mailman-Version: 2.1.29 Precedence: list List-Id: Proxmox Backup Server development discussion List-Unsubscribe: , List-Archive: List-Post: List-Help: List-Subscribe: , X-List-Received-Date: Mon, 13 Sep 2021 14:19:06 -0000 similar to pve/pmg, a user can call the api with this utility without going through the proxy/daemon, as well as list the api endpoints (with child links) and get the api description of endpoints this is mainly intended for debugging, but it is also useful for situations where some api calls do not have an equivalent in a binary and a user does not want to go through the api not implemented are the http2 api calls (since it is a separate api and it wouldn't be that easy to do) there are a few quirks though, related to the 'ls' command: i extract the 'child-link' from the property name of the 'match_all' statement of the router, but this does not always match with the property from the relevant 'get' api call so it fails there (e.g. /tape/drive ) this can be fixed in the respective api calls (e.g. by renaming the parameter that comes from the path) includes bash/zsh completion helpers and a basic manpage Signed-off-by: Dominik Csapak --- Makefile | 2 + debian/pbs-shell.bc | 3 + debian/proxmox-backup-server.bash-completion | 1 + debian/proxmox-backup-server.install | 3 + docs/Makefile | 8 + docs/pbs-shell/description.rst | 3 + docs/pbs-shell/man1.rst | 40 ++ src/api2/node/tasks.rs | 2 +- src/bin/pbs-shell.rs | 528 +++++++++++++++++++ zsh-completions/_pbs-shell | 13 + 10 files changed, 602 insertions(+), 1 deletion(-) create mode 100644 debian/pbs-shell.bc create mode 100644 docs/pbs-shell/description.rst create mode 100644 docs/pbs-shell/man1.rst create mode 100644 src/bin/pbs-shell.rs create mode 100644 zsh-completions/_pbs-shell diff --git a/Makefile b/Makefile index c1aecf61..abeaff37 100644 --- a/Makefile +++ b/Makefile @@ -11,6 +11,7 @@ USR_BIN := \ proxmox-backup-client \ proxmox-file-restore \ pxar \ + pbs-shell \ proxmox-tape \ pmtx \ pmt @@ -172,6 +173,7 @@ $(COMPILED_BINS) $(COMPILEDIR)/dump-catalog-shell-cli $(COMPILEDIR)/docgen: .do- --bin proxmox-backup-api \ --bin proxmox-backup-proxy \ --bin proxmox-backup-manager \ + --bin pbs-shell \ --bin docgen $(CARGO) build $(CARGO_BUILD_ARGS) \ --package proxmox-backup-banner \ diff --git a/debian/pbs-shell.bc b/debian/pbs-shell.bc new file mode 100644 index 00000000..3d17187c --- /dev/null +++ b/debian/pbs-shell.bc @@ -0,0 +1,3 @@ +# pbs-shell bash completion + +complete -C 'pbs-shell bashcomplete' pbs-shell diff --git a/debian/proxmox-backup-server.bash-completion b/debian/proxmox-backup-server.bash-completion index a2165699..8d6a7047 100644 --- a/debian/proxmox-backup-server.bash-completion +++ b/debian/proxmox-backup-server.bash-completion @@ -2,3 +2,4 @@ debian/proxmox-backup-manager.bc proxmox-backup-manager debian/proxmox-tape.bc proxmox-tape debian/pmtx.bc pmtx debian/pmt.bc pmt +debian/pbs-shell.bc pbs-shell diff --git a/debian/proxmox-backup-server.install b/debian/proxmox-backup-server.install index 6e2219b4..5e1071fa 100644 --- a/debian/proxmox-backup-server.install +++ b/debian/proxmox-backup-server.install @@ -14,6 +14,7 @@ usr/sbin/proxmox-backup-manager usr/bin/pmtx usr/bin/pmt usr/bin/proxmox-tape +usr/bin/pbs-shell usr/share/javascript/proxmox-backup/index.hbs usr/share/javascript/proxmox-backup/css/ext6-pbs.css usr/share/javascript/proxmox-backup/images @@ -24,6 +25,7 @@ usr/share/man/man1/proxmox-backup-proxy.1 usr/share/man/man1/proxmox-tape.1 usr/share/man/man1/pmtx.1 usr/share/man/man1/pmt.1 +usr/share/man/man1/pbs-shell.1 usr/share/man/man5/acl.cfg.5 usr/share/man/man5/datastore.cfg.5 usr/share/man/man5/user.cfg.5 @@ -38,3 +40,4 @@ usr/share/zsh/vendor-completions/_proxmox-backup-manager usr/share/zsh/vendor-completions/_proxmox-tape usr/share/zsh/vendor-completions/_pmtx usr/share/zsh/vendor-completions/_pmt +usr/share/zsh/vendor-completions/_pbs-shell diff --git a/docs/Makefile b/docs/Makefile index 5e37f7d1..e67df2ea 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -10,6 +10,7 @@ GENERATED_SYNOPSIS := \ pxar/synopsis.rst \ pmtx/synopsis.rst \ pmt/synopsis.rst \ + pbs-shell/synopsis.rst \ config/media-pool/config.rst \ config/tape/config.rst \ config/tape-job/config.rst \ @@ -24,6 +25,7 @@ MAN1_PAGES := \ pxar.1 \ pmtx.1 \ pmt.1 \ + pbs-shell.1 \ proxmox-tape.1 \ proxmox-backup-proxy.1 \ proxmox-backup-client.1 \ @@ -117,6 +119,12 @@ pmt/synopsis.rst: ${COMPILEDIR}/pmt pmt.1: pmt/man1.rst pmt/description.rst pmt/options.rst pmt/synopsis.rst rst2man $< >$@ +pbs-shell/synopsis.rst: ${COMPILEDIR}/pbs-shell + ${COMPILEDIR}/pbs-shell printdoc > pbs-shell/synopsis.rst + +pbs-shell.1: pbs-shell/man1.rst pbs-shell/description.rst pbs-shell/synopsis.rst + rst2man $< >$@ + config/datastore/config.rst: ${COMPILEDIR}/docgen ${COMPILEDIR}/docgen datastore.cfg >$@ diff --git a/docs/pbs-shell/description.rst b/docs/pbs-shell/description.rst new file mode 100644 index 00000000..8dfcae15 --- /dev/null +++ b/docs/pbs-shell/description.rst @@ -0,0 +1,3 @@ +The ``pbs-shell`` command can show and execute api calls and their parameters. +It is mainly intended for use during debugging. + diff --git a/docs/pbs-shell/man1.rst b/docs/pbs-shell/man1.rst new file mode 100644 index 00000000..d0a1d07b --- /dev/null +++ b/docs/pbs-shell/man1.rst @@ -0,0 +1,40 @@ +========================== +pbs-shell +========================== + +.. include:: ../epilog.rst + +------------------------------------------------------------- +Show and execute PBS API calls +------------------------------------------------------------- + +:Author: |AUTHOR| +:Version: Version |VERSION| +:Manual section: 1 + + +Synopsis +========== + +.. include:: synopsis.rst + + +Common Options +============== + +Commands generating output supports the ``--output-format`` +parameter. It accepts the following values: + +:``text``: Text format (default). Human readable. + +:``json``: JSON (single line). + +:``json-pretty``: JSON (multiple lines, nicely formatted). + + +Description +============ + +.. include:: description.rst + +.. include:: ../pbs-copyright.rst diff --git a/src/api2/node/tasks.rs b/src/api2/node/tasks.rs index 9aaf6f1a..e422a974 100644 --- a/src/api2/node/tasks.rs +++ b/src/api2/node/tasks.rs @@ -258,7 +258,7 @@ fn extract_upid(param: &Value) -> Result { }, )] /// Read task log. -async fn read_task_log( +pub async fn read_task_log( param: Value, mut rpcenv: &mut dyn RpcEnvironment, ) -> Result { diff --git a/src/bin/pbs-shell.rs b/src/bin/pbs-shell.rs new file mode 100644 index 00000000..a9f5ad29 --- /dev/null +++ b/src/bin/pbs-shell.rs @@ -0,0 +1,528 @@ +use anyhow::{bail, format_err, Error}; +use hyper::Method; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use tokio::signal::unix::{signal, SignalKind}; +use futures::FutureExt; + +use std::collections::HashMap; + +use proxmox::api::{ + api, + cli::*, + format::DocumentationFormat, + schema::{parse_parameter_strings, ApiType, ParameterSchema, Schema}, + ApiHandler, ApiMethod, RpcEnvironment, SubRoute, +}; + +use pbs_api_types::{UPID, PROXMOX_UPID_REGEX}; + +const PROG_NAME: &str = "pbs-shell"; + +fn complete_api_path(complete_me: &str, _map: &HashMap) -> Vec { + pbs_runtime::main(async { complete_api_path_do(complete_me, None).await }) +} + +fn complete_api_path_get(complete_me: &str, _map: &HashMap) -> Vec { + pbs_runtime::main(async { complete_api_path_do(complete_me, Some("r")).await }) +} + +fn complete_api_path_set(complete_me: &str, _map: &HashMap) -> Vec { + pbs_runtime::main(async { complete_api_path_do(complete_me, Some("w")).await }) +} + +fn complete_api_path_create(complete_me: &str, _map: &HashMap) -> Vec { + pbs_runtime::main(async { complete_api_path_do(complete_me, Some("c")).await }) +} + +fn complete_api_path_delete(complete_me: &str, _map: &HashMap) -> Vec { + pbs_runtime::main(async { complete_api_path_do(complete_me, Some("d")).await }) +} + +fn complete_api_path_ls(complete_me: &str, _map: &HashMap) -> Vec { + pbs_runtime::main(async { complete_api_path_do(complete_me, Some("D")).await }) +} + +async fn complete_api_path_do(mut complete_me: &str, capability: Option<&str>) -> Vec { + if complete_me.is_empty() { + complete_me = "/"; + } + + let mut list = Vec::new(); + + let mut lookup_path = complete_me.to_string(); + let mut filter = ""; + let last_path_index = complete_me.rfind('/'); + if let Some(index) = last_path_index { + if index != complete_me.len() - 1 { + lookup_path = complete_me[..(index + 1)].to_string(); + if index < complete_me.len() - 1 { + filter = &complete_me[(index + 1)..]; + } + } + } + + let uid = nix::unistd::Uid::current(); + + let username = match nix::unistd::User::from_uid(uid) { + Ok(Some(user)) => user.name, + _ => "root@pam".to_string(), + }; + let mut rpcenv = CliEnvironment::new(); + rpcenv.set_auth_id(Some(format!("{}@pam", username))); + + while let Ok(children) = get_api_children(lookup_path.clone(), &mut rpcenv).await { + let old_len = list.len(); + for entry in children { + let name = entry.name; + let caps = entry.capabilities; + + if filter.is_empty() || name.starts_with(filter) { + let mut path = format!("{}{}", lookup_path, name); + if caps.contains('D') { + path.push('/'); + list.push(path.clone()); + } else if let Some(cap) = capability { + if caps.contains(cap) { + list.push(path); + } + } else { + list.push(path); + } + } + } + + if list.len() == 1 && old_len != 1 && list[0].ends_with('/') { + // we added only one match and it was a directory, lookup again + lookup_path = list[0].clone(); + filter = ""; + continue; + } + + break; + } + + list +} + +async fn get_child_links( + path: &str, + rpcenv: &mut dyn RpcEnvironment, +) -> Result, Error> { + let mut uri_param = HashMap::new(); + let (path, components) = proxmox_backup::tools::normalize_uri_path(&path)?; + + let info = &proxmox_backup::api2::ROUTER + .find_route(&components, &mut uri_param) + .ok_or_else(|| format_err!("no such resource"))?; + + match info.subroute { + Some(SubRoute::Map(map)) => Ok(map.iter().map(|(name, _)| name.to_string()).collect()), + Some(SubRoute::MatchAll { param_name, .. }) => { + let get_call = info.get.ok_or_else(|| format_err!("no such resource"))?; + let list = call_api(get_call, rpcenv, serde_json::to_value(uri_param)?).await?; + Ok(list + .as_array() + .ok_or_else(|| format_err!("{} did not return an array", path))? + .iter() + .map(|item| { + item[param_name] + .as_str() + .map(|c| c.to_string()) + .ok_or_else(|| format_err!("no such property {}", param_name)) + }) + .collect::, _>>()?) + } + None => bail!("link does not define child links"), + } +} + +fn get_api_method( + method: Method, + path: &str, +) -> Result<(&'static ApiMethod, HashMap), Error> { + let mut uri_param = HashMap::new(); + let (path, components) = proxmox_backup::tools::normalize_uri_path(&path)?; + if let Some(method) = + &proxmox_backup::api2::ROUTER.find_method(&components, method.clone(), &mut uri_param) + { + Ok((method, uri_param)) + } else { + bail!("no {} handler defined for '{}'", method, path); + } +} + +fn merge_parameters( + uri_param: HashMap, + param: Value, + schema: ParameterSchema, +) -> Result { + let mut param_list: Vec<(String, String)> = vec![]; + + for (k, v) in uri_param { + param_list.push((k.clone(), v.clone())); + } + + if let Some(map) = param.as_object() { + for (k, v) in map { + param_list.push((k.clone(), v.as_str().unwrap().to_string())); + } + } + + let params = parse_parameter_strings(¶m_list, schema, true)?; + + Ok(params) +} + +async fn call_api( + method: &'static ApiMethod, + rpcenv: &mut dyn RpcEnvironment, + params: Value, +) -> Result { + match method.handler { + ApiHandler::AsyncHttp(_handler) => { + bail!("not implemented"); + } + ApiHandler::Sync(handler) => (handler)(params, method, rpcenv), + ApiHandler::Async(handler) => (handler)(params, method, rpcenv).await, + } +} + +async fn handle_worker(upid_str: &str) -> Result<(), Error> { + + let upid: UPID = upid_str.parse()?; + let mut signal_stream = signal(SignalKind::interrupt())?; + let abort_future = async move { + while signal_stream.recv().await.is_some() { + println!("got shutdown request (SIGINT)"); + proxmox_backup::server::abort_local_worker(upid.clone()); + } + Ok::<_, Error>(()) + }; + + let result_future = proxmox_backup::server::wait_for_local_worker(upid_str); + + futures::select!{ + result = result_future.fuse() => result?, + abort = abort_future.fuse() => abort?, + }; + + Ok(()) +} + +async fn call_api_and_format_result( + method: Method, + path: String, + mut param: Value, + rpcenv: &mut dyn RpcEnvironment, +) -> Result<(), Error> { + let mut output_format = extract_output_format(&mut param); + let (method, uri_param) = get_api_method(method, &path)?; + let params = merge_parameters(uri_param, param, method.parameters)?; + + let mut result = call_api(method, rpcenv, params).await?; + + if let Some(upid) = result.as_str() { + if PROXMOX_UPID_REGEX.is_match(upid) { + handle_worker(upid).await?; + + if output_format == "text" { + return Ok(()); + } + } + } + + let options = default_table_format_options(); + let return_type = &method.returns; + if matches!(return_type.schema, Schema::Null) { + output_format = "json-pretty".to_string(); + } + + format_and_print_result_full(&mut result, return_type, &output_format, &options); + + Ok(()) +} + +#[api( + input: { + additional_properties: true, + properties: { + "api-path": { + type: String, + description: "API path.", + }, + "output-format": { + schema: OUTPUT_FORMAT, + optional: true, + }, + }, + }, +)] +/// Call API PUT on +async fn set(api_path: String, param: Value, rpcenv: &mut dyn RpcEnvironment) -> Result<(), Error> { + call_api_and_format_result(Method::PUT, api_path, param, rpcenv).await +} + +#[api( + input: { + additional_properties: true, + properties: { + "api-path": { + type: String, + description: "API path.", + }, + "output-format": { + schema: OUTPUT_FORMAT, + optional: true, + }, + }, + }, +)] +/// Call API POST on +async fn create(api_path: String, param: Value, rpcenv: &mut dyn RpcEnvironment) -> Result<(), Error> { + call_api_and_format_result(Method::POST, api_path, param, rpcenv).await +} + +#[api( + input: { + additional_properties: true, + properties: { + "api-path": { + type: String, + description: "API path.", + }, + "output-format": { + schema: OUTPUT_FORMAT, + optional: true, + }, + }, + }, +)] +/// Call API GET on +async fn get(api_path: String, param: Value, rpcenv: &mut dyn RpcEnvironment) -> Result<(), Error> { + call_api_and_format_result(Method::GET, api_path, param, rpcenv).await +} + +#[api( + input: { + additional_properties: true, + properties: { + "api-path": { + type: String, + description: "API path.", + }, + "output-format": { + schema: OUTPUT_FORMAT, + optional: true, + }, + }, + }, +)] +/// Call API DELETE on +async fn delete(api_path: String, param: Value, rpcenv: &mut dyn RpcEnvironment) -> Result<(), Error> { + call_api_and_format_result(Method::DELETE, api_path, param, rpcenv).await +} + +#[api( + input: { + properties: { + path: { + type: String, + description: "API path.", + }, + verbose: { + type: Boolean, + description: "Verbose output format.", + optional: true, + default: false, + } + }, + }, +)] +/// Get API usage information for +async fn usage( + path: String, + verbose: bool, + _param: Value, + _rpcenv: &mut dyn RpcEnvironment, +) -> Result<(), Error> { + let docformat = if verbose { + DocumentationFormat::Full + } else { + DocumentationFormat::Short + }; + let mut found = false; + for command in &["get", "set", "create", "delete"] { + let http_method = match *command { + "get" => Method::GET, + "set" => Method::PUT, + "create" => Method::POST, + "delete" => Method::DELETE, + _ => unreachable!(), + }; + let (info, uri_params) = match get_api_method(http_method, &path) { + Ok(some) => some, + Err(_) => continue, + }; + found = true; + + let skip_params: Vec<&str> = uri_params.keys().map(|s| &**s).collect(); + + let cmd = CliCommand::new(info); + let prefix = format!("USAGE: {} {} {}", PROG_NAME, command, path); + + print!( + "{}", + generate_usage_str(&prefix, &cmd, docformat, "", &skip_params) + ); + } + + if !found { + bail!("no such resource '{}'", path); + } + Ok(()) +} + +#[api()] +#[derive(Debug, Serialize, Deserialize)] +/// A child link with capabilities +struct ApiDirEntry { + /// The name of the link + name: String, + /// The capabilities of the path (format Drwcd) + capabilities: String, +} + +const LS_SCHEMA: &proxmox::api::schema::Schema = + &proxmox::api::schema::ArraySchema::new("List of child links", &ApiDirEntry::API_SCHEMA) + .schema(); + +async fn get_api_children( + path: String, + rpcenv: &mut dyn RpcEnvironment, +) -> Result, Error> { + let mut res = Vec::new(); + for link in get_child_links(&path, rpcenv).await? { + let path = format!("{}/{}", path, link); + let (path, _) = proxmox_backup::tools::normalize_uri_path(&path)?; + let mut cap = String::new(); + + if get_child_links(&path, rpcenv).await.is_ok() { + cap.push('D'); + } else { + cap.push('-'); + } + + let cap_list = &[ + (Method::GET, 'r'), + (Method::PUT, 'w'), + (Method::POST, 'c'), + (Method::DELETE, 'd'), + ]; + + for (method, c) in cap_list { + if get_api_method(method.clone(), &path).is_ok() { + cap.push(*c); + } else { + cap.push('-'); + } + } + + res.push(ApiDirEntry { + name: link.to_string(), + capabilities: cap, + }); + } + + Ok(res) +} + +#[api( + input: { + properties: { + path: { + type: String, + description: "API path.", + }, + "output-format": { + schema: OUTPUT_FORMAT, + optional: true, + }, + }, + }, +)] +/// Get API usage information for +async fn ls(path: String, mut param: Value, rpcenv: &mut dyn RpcEnvironment) -> Result<(), Error> { + let output_format = extract_output_format(&mut param); + + let options = TableFormatOptions::new() + .noborder(true) + .noheader(true) + .sortby("name", false); + + let res = get_api_children(path, rpcenv).await?; + + format_and_print_result_full( + &mut serde_json::to_value(res)?, + &proxmox::api::schema::ReturnType { + optional: false, + schema: &LS_SCHEMA, + }, + &output_format, + &options, + ); + + Ok(()) +} + +fn main() -> Result<(), Error> { + let cmd_def = CliCommandMap::new() + .insert( + "get", + CliCommand::new(&API_METHOD_GET) + .arg_param(&["api-path"]) + .completion_cb("api-path", complete_api_path_get), + ) + .insert( + "set", + CliCommand::new(&API_METHOD_SET) + .arg_param(&["api-path"]) + .completion_cb("api-path", complete_api_path_set), + ) + .insert( + "create", + CliCommand::new(&API_METHOD_CREATE) + .arg_param(&["api-path"]) + .completion_cb("api-path", complete_api_path_create), + ) + .insert( + "delete", + CliCommand::new(&API_METHOD_DELETE) + .arg_param(&["api-path"]) + .completion_cb("api-path", complete_api_path_delete), + ) + .insert( + "ls", + CliCommand::new(&API_METHOD_LS) + .arg_param(&["path"]) + .completion_cb("path", complete_api_path_ls), + ) + .insert( + "usage", + CliCommand::new(&API_METHOD_USAGE) + .arg_param(&["path"]) + .completion_cb("path", complete_api_path), + ); + + let uid = nix::unistd::Uid::current(); + + let username = match nix::unistd::User::from_uid(uid)? { + Some(user) => user.name, + None => bail!("unable to get user name"), + }; + let mut rpcenv = CliEnvironment::new(); + rpcenv.set_auth_id(Some(format!("{}@pam", username))); + + pbs_runtime::main(run_async_cli_command(cmd_def, rpcenv)); + Ok(()) +} diff --git a/zsh-completions/_pbs-shell b/zsh-completions/_pbs-shell new file mode 100644 index 00000000..507f15ae --- /dev/null +++ b/zsh-completions/_pbs-shell @@ -0,0 +1,13 @@ +#compdef _pbs-shell() pbs-shell + +function _pbs-shell() { + local cwords line point cmd curr prev + cwords=${#words[@]} + line=$words + point=${#line} + cmd=${words[1]} + curr=${words[cwords]} + prev=${words[cwords-1]} + compadd -- $(COMP_CWORD="$cwords" COMP_LINE="$line" COMP_POINT="$point" \ + pbs-shell bashcomplete "$cmd" "$curr" "$prev") +} -- 2.30.2