diff --git a/CHANGELOG.md b/CHANGELOG.md index 1344546..4b65e63 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,9 +27,13 @@ Released on ?? +- **Added support for S3 compatible backends** + - Changed `AWS S3` to `S3` in ui + - Added new `endpoint` and `new-path-style` to s3 connection parameters - Bugfix: - [Issue 92](https://github.com/veeso/termscp/issues/92): updated ssh2-config to 0.1.3, which solves this issue. - Dependencies: + - Updated `remotefs-rs-aws-s3` to `0.2.0` - Updated `tui-realm-stdlib` to `1.1.6` ## 0.8.0 diff --git a/Cargo.lock b/Cargo.lock index 20295ec..d820404 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1794,9 +1794,9 @@ dependencies = [ [[package]] name = "remotefs-aws-s3" -version = "0.1.0" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36b130b79d6ade3821554c46a87fd54dea024255daa5ecac5e9f68e2a4d3230a" +checksum = "9939849e0b3895b07f258458d0fa5cd4632c78ffc517e66e1f758ddf23990542" dependencies = [ "chrono", "log", diff --git a/Cargo.toml b/Cargo.toml index d708cdc..91150e2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -48,7 +48,7 @@ open = "2.0.2" rand = "0.8.4" regex = "1.5.4" remotefs = "^0.2.0" -remotefs-aws-s3 = "^0.1.0" +remotefs-aws-s3 = "^0.2.0" remotefs-ftp = { version = "^0.1.0", features = [ "secure" ] } remotefs-ssh = "^0.1.0" rpassword = "5.0.1" diff --git a/src/config/bookmarks.rs b/src/config/bookmarks.rs index 7c76af2..954be4f 100644 --- a/src/config/bookmarks.rs +++ b/src/config/bookmarks.rs @@ -64,10 +64,13 @@ pub struct Bookmark { #[derive(Clone, Deserialize, Serialize, Debug, PartialEq, Default)] pub struct S3Params { pub bucket: String, - pub region: String, + pub region: Option, + pub endpoint: Option, pub profile: Option, pub access_key: Option, pub secret_access_key: Option, + /// NOTE: there are no session token and security token since they are always temporary + pub new_path_style: Option, } // -- impls @@ -123,9 +126,11 @@ impl From for S3Params { S3Params { bucket: params.bucket_name, region: params.region, + endpoint: params.endpoint, profile: params.profile, access_key: params.access_key, secret_access_key: params.secret_access_key, + new_path_style: Some(params.new_path_style), } } } @@ -133,8 +138,10 @@ impl From for S3Params { impl From for AwsS3Params { fn from(params: S3Params) -> Self { AwsS3Params::new(params.bucket, params.region, params.profile) + .endpoint(params.endpoint) .access_key(params.access_key) .secret_access_key(params.secret_access_key) + .new_path_style(params.new_path_style.unwrap_or(false)) } } @@ -234,7 +241,7 @@ mod tests { #[test] fn bookmark_from_s3_ftparams() { let params = ProtocolParams::AwsS3( - AwsS3Params::new("omar", "eu-west-1", Some("test")) + AwsS3Params::new("omar", Some("eu-west-1"), Some("test")) .access_key(Some("pippo")) .secret_access_key(Some("pluto")), ); @@ -248,7 +255,7 @@ mod tests { assert!(bookmark.password.is_none()); let s3: &S3Params = bookmark.s3.as_ref().unwrap(); assert_eq!(s3.bucket.as_str(), "omar"); - assert_eq!(s3.region.as_str(), "eu-west-1"); + assert_eq!(s3.region.as_deref().unwrap(), "eu-west-1"); assert_eq!(s3.profile.as_deref().unwrap(), "test"); assert_eq!(s3.access_key.as_deref().unwrap(), "pippo"); assert_eq!(s3.secret_access_key.as_deref().unwrap(), "pluto"); @@ -283,19 +290,23 @@ mod tests { password: None, s3: Some(S3Params { bucket: String::from("veeso"), - region: String::from("eu-west-1"), + region: Some(String::from("eu-west-1")), + endpoint: Some(String::from("omar")), profile: Some(String::from("default")), access_key: Some(String::from("pippo")), secret_access_key: Some(String::from("pluto")), + new_path_style: Some(true), }), }; let params = FileTransferParams::from(bookmark); assert_eq!(params.protocol, FileTransferProtocol::AwsS3); let gparams = params.params.s3_params().unwrap(); assert_eq!(gparams.bucket_name.as_str(), "veeso"); - assert_eq!(gparams.region.as_str(), "eu-west-1"); + assert_eq!(gparams.region.as_deref().unwrap(), "eu-west-1"); + assert_eq!(gparams.endpoint.as_deref().unwrap(), "omar"); assert_eq!(gparams.profile.as_deref().unwrap(), "default"); assert_eq!(gparams.access_key.as_deref().unwrap(), "pippo"); assert_eq!(gparams.secret_access_key.as_deref().unwrap(), "pluto"); + assert_eq!(gparams.new_path_style, true); } } diff --git a/src/config/serialization.rs b/src/config/serialization.rs index 038f4c1..8ead7ae 100644 --- a/src/config/serialization.rs +++ b/src/config/serialization.rs @@ -419,10 +419,12 @@ mod tests { assert_eq!(host.protocol, FileTransferProtocol::AwsS3); let s3 = host.s3.as_ref().unwrap(); assert_eq!(s3.bucket.as_str(), "veeso"); - assert_eq!(s3.region.as_str(), "eu-west-1"); + assert_eq!(s3.region.as_deref().unwrap(), "eu-west-1"); assert_eq!(s3.profile.as_deref().unwrap(), "default"); + assert_eq!(s3.endpoint.as_deref().unwrap(), "http://localhost:9000"); assert_eq!(s3.access_key.as_deref().unwrap(), "pippo"); assert_eq!(s3.secret_access_key.as_deref().unwrap(), "pluto"); + assert_eq!(s3.new_path_style.unwrap(), true); } #[test] @@ -470,10 +472,12 @@ mod tests { password: None, s3: Some(S3Params { bucket: "veeso".to_string(), - region: "eu-west-1".to_string(), + region: Some("eu-west-1".to_string()), + endpoint: None, profile: None, access_key: None, secret_access_key: None, + new_path_style: None, }), }, ); @@ -534,9 +538,11 @@ mod tests { [bookmarks.my-bucket.s3] bucket = "veeso" region = "eu-west-1" + endpoint = "http://localhost:9000" profile = "default" access_key = "pippo" secret_access_key = "pluto" + new_path_style = true [recents] ISO20201215T094000Z = { address = "172.16.104.10", port = 22, protocol = "SCP", username = "root" } diff --git a/src/filetransfer/builder.rs b/src/filetransfer/builder.rs index aa906bb..1135155 100644 --- a/src/filetransfer/builder.rs +++ b/src/filetransfer/builder.rs @@ -73,10 +73,16 @@ impl Builder { /// Build aws s3 client from parameters fn aws_s3_client(params: AwsS3Params) -> AwsS3Fs { - let mut client = AwsS3Fs::new(params.bucket_name, params.region); + let mut client = AwsS3Fs::new(params.bucket_name).new_path_style(params.new_path_style); + if let Some(region) = params.region { + client = client.region(region); + } if let Some(profile) = params.profile { client = client.profile(profile); } + if let Some(endpoint) = params.endpoint { + client = client.endpoint(endpoint); + } if let Some(access_key) = params.access_key { client = client.access_key(access_key); } @@ -151,7 +157,9 @@ mod test { #[test] fn should_build_aws_s3_fs() { let params = ProtocolParams::AwsS3( - AwsS3Params::new("omar", "eu-west-1", Some("test")) + AwsS3Params::new("omar", Some("eu-west-1"), Some("test")) + .endpoint(Some("http://localhost:9000")) + .new_path_style(true) .access_key(Some("pippo")) .secret_access_key(Some("pluto")) .security_token(Some("omar")) diff --git a/src/filetransfer/params.rs b/src/filetransfer/params.rs index 0168367..3838f53 100644 --- a/src/filetransfer/params.rs +++ b/src/filetransfer/params.rs @@ -59,12 +59,14 @@ pub struct GenericProtocolParams { #[derive(Debug, Clone)] pub struct AwsS3Params { pub bucket_name: String, - pub region: String, + pub region: Option, + pub endpoint: Option, pub profile: Option, pub access_key: Option, pub secret_access_key: Option, pub security_token: Option, pub session_token: Option, + pub new_path_style: bool, } impl FileTransferParams { @@ -167,18 +169,26 @@ impl GenericProtocolParams { impl AwsS3Params { /// Instantiates a new `AwsS3Params` struct - pub fn new>(bucket: S, region: S, profile: Option) -> Self { + pub fn new>(bucket: S, region: Option, profile: Option) -> Self { Self { bucket_name: bucket.as_ref().to_string(), - region: region.as_ref().to_string(), + region: region.map(|x| x.as_ref().to_string()), profile: profile.map(|x| x.as_ref().to_string()), + endpoint: None, access_key: None, secret_access_key: None, security_token: None, session_token: None, + new_path_style: false, } } + /// Construct aws s3 params with specified endpoint + pub fn endpoint>(mut self, endpoint: Option) -> Self { + self.endpoint = endpoint.map(|x| x.as_ref().to_string()); + self + } + /// Construct aws s3 params with provided access key pub fn access_key>(mut self, key: Option) -> Self { self.access_key = key.map(|x| x.as_ref().to_string()); @@ -202,6 +212,12 @@ impl AwsS3Params { self.session_token = key.map(|x| x.as_ref().to_string()); self } + + /// Specify new path style when constructing aws s3 params + pub fn new_path_style(mut self, new_path_style: bool) -> Self { + self.new_path_style = new_path_style; + self + } } #[cfg(test)] @@ -240,35 +256,42 @@ mod test { #[test] fn should_init_aws_s3_params() { - let params: AwsS3Params = AwsS3Params::new("omar", "eu-west-1", Some("test")); + let params: AwsS3Params = AwsS3Params::new("omar", Some("eu-west-1"), Some("test")); assert_eq!(params.bucket_name.as_str(), "omar"); - assert_eq!(params.region.as_str(), "eu-west-1"); + assert_eq!(params.region.as_deref().unwrap(), "eu-west-1"); assert_eq!(params.profile.as_deref().unwrap(), "test"); + assert!(params.endpoint.is_none()); assert!(params.access_key.is_none()); assert!(params.secret_access_key.is_none()); assert!(params.security_token.is_none()); assert!(params.session_token.is_none()); + assert_eq!(params.new_path_style, false); } #[test] fn should_init_aws_s3_params_with_optionals() { - let params: AwsS3Params = AwsS3Params::new("omar", "eu-west-1", Some("test")) + let params: AwsS3Params = AwsS3Params::new("omar", Some("eu-west-1"), Some("test")) + .endpoint(Some("http://omar.it")) .access_key(Some("pippo")) .secret_access_key(Some("pluto")) .security_token(Some("omar")) - .session_token(Some("gerry-scotti")); + .session_token(Some("gerry-scotti")) + .new_path_style(true); assert_eq!(params.bucket_name.as_str(), "omar"); - assert_eq!(params.region.as_str(), "eu-west-1"); + assert_eq!(params.region.as_deref().unwrap(), "eu-west-1"); assert_eq!(params.profile.as_deref().unwrap(), "test"); + assert_eq!(params.endpoint.as_deref().unwrap(), "http://omar.it"); assert_eq!(params.access_key.as_deref().unwrap(), "pippo"); assert_eq!(params.secret_access_key.as_deref().unwrap(), "pluto"); assert_eq!(params.security_token.as_deref().unwrap(), "omar"); assert_eq!(params.session_token.as_deref().unwrap(), "gerry-scotti"); + assert_eq!(params.new_path_style, true); } #[test] fn references() { - let mut params = ProtocolParams::AwsS3(AwsS3Params::new("omar", "eu-west-1", Some("test"))); + let mut params = + ProtocolParams::AwsS3(AwsS3Params::new("omar", Some("eu-west-1"), Some("test"))); assert!(params.s3_params().is_some()); assert!(params.generic_params().is_none()); assert!(params.mut_generic_params().is_none()); diff --git a/src/system/bookmarks_client.rs b/src/system/bookmarks_client.rs index 132985c..fa1dd07 100644 --- a/src/system/bookmarks_client.rs +++ b/src/system/bookmarks_client.rs @@ -506,7 +506,7 @@ mod tests { assert_eq!(params.profile.as_deref().unwrap(), "test"); assert_eq!(params.secret_access_key.as_deref().unwrap(), "pluto"); assert_eq!(params.bucket_name.as_str(), "omar"); - assert_eq!(params.region.as_str(), "eu-west-1"); + assert_eq!(params.region.as_deref().unwrap(), "eu-west-1"); } #[test] @@ -524,7 +524,7 @@ mod tests { let params = bookmark.params.s3_params().unwrap(); assert_eq!(params.profile.as_deref().unwrap(), "test"); assert_eq!(params.bucket_name.as_str(), "omar"); - assert_eq!(params.region.as_str(), "eu-west-1"); + assert_eq!(params.region.as_deref().unwrap(), "eu-west-1"); // secrets assert_eq!(params.access_key, None); assert_eq!(params.secret_access_key, None); @@ -546,7 +546,7 @@ mod tests { let params = bookmark.params.s3_params().unwrap(); assert_eq!(params.profile.as_deref().unwrap(), "test"); assert_eq!(params.bucket_name.as_str(), "omar"); - assert_eq!(params.region.as_str(), "eu-west-1"); + assert_eq!(params.region.as_deref().unwrap(), "eu-west-1"); // secrets assert_eq!(params.access_key, None); assert_eq!(params.secret_access_key, None); @@ -849,7 +849,9 @@ mod tests { FileTransferParams::new( FileTransferProtocol::AwsS3, ProtocolParams::AwsS3( - AwsS3Params::new("omar", "eu-west-1", Some("test")) + AwsS3Params::new("omar", Some("eu-west-1"), Some("test")) + .endpoint(Some("http://localhost:9000")) + .new_path_style(false) .access_key(Some("pippo")) .secret_access_key(Some("pluto")) .security_token(Some("omar")) diff --git a/src/ui/activities/auth/bookmarks.rs b/src/ui/activities/auth/bookmarks.rs index 4f89b8f..2549903 100644 --- a/src/ui/activities/auth/bookmarks.rs +++ b/src/ui/activities/auth/bookmarks.rs @@ -227,11 +227,12 @@ impl AuthActivity { fn load_bookmark_s3_into_gui(&mut self, params: AwsS3Params) { self.mount_s3_bucket(params.bucket_name.as_str()); - self.mount_s3_region(params.region.as_str()); + self.mount_s3_region(params.region.as_deref().unwrap_or("")); self.mount_s3_profile(params.profile.as_deref().unwrap_or("")); self.mount_s3_access_key(params.access_key.as_deref().unwrap_or("")); self.mount_s3_secret_access_key(params.secret_access_key.as_deref().unwrap_or("")); self.mount_s3_security_token(params.security_token.as_deref().unwrap_or("")); self.mount_s3_session_token(params.session_token.as_deref().unwrap_or("")); + // TODO: add mount } } diff --git a/src/ui/activities/auth/misc.rs b/src/ui/activities/auth/misc.rs index a00d2dc..e9d7169 100644 --- a/src/ui/activities/auth/misc.rs +++ b/src/ui/activities/auth/misc.rs @@ -88,9 +88,6 @@ impl AuthActivity { if params.bucket_name.is_empty() { return Err("Invalid bucket"); } - if params.region.is_empty() { - return Err("Invalid region"); - } Ok(FileTransferParams { protocol: FileTransferProtocol::AwsS3, params: ProtocolParams::AwsS3(params), diff --git a/src/ui/activities/auth/view.rs b/src/ui/activities/auth/view.rs index 322dcd0..368b883 100644 --- a/src/ui/activities/auth/view.rs +++ b/src/ui/activities/auth/view.rs @@ -726,12 +726,13 @@ impl AuthActivity { /// Collect s3 input values from view pub(super) fn get_s3_params_input(&self) -> AwsS3Params { let bucket: String = self.get_input_s3_bucket(); - let region: String = self.get_input_s3_region(); + let region: Option = self.get_input_s3_region(); let profile: Option = self.get_input_s3_profile(); let access_key = self.get_input_s3_access_key(); let secret_access_key = self.get_input_s3_secret_access_key(); let security_token = self.get_input_s3_security_token(); let session_token = self.get_input_s3_session_token(); + // TODO: collect AwsS3Params::new(bucket, region, profile) .access_key(access_key) .secret_access_key(secret_access_key) @@ -777,10 +778,10 @@ impl AuthActivity { } } - pub(super) fn get_input_s3_region(&self) -> String { + pub(super) fn get_input_s3_region(&self) -> Option { match self.app.state(&Id::S3Region) { - Ok(State::One(StateValue::String(x))) => x, - _ => String::new(), + Ok(State::One(StateValue::String(x))) if !x.is_empty() => Some(x), + _ => None, } } @@ -863,8 +864,12 @@ impl AuthActivity { None => String::default(), }; format!( - "{}://{} ({}) {}", - protocol, s3.bucket_name, s3.region, profile + "{}://{}{} ({}) {}", + protocol, + s3.endpoint.unwrap_or_default(), + s3.bucket_name, + s3.region.as_deref().unwrap_or("custom"), + profile ) } ProtocolParams::Generic(params) => { diff --git a/src/ui/activities/filetransfer/misc.rs b/src/ui/activities/filetransfer/misc.rs index 5c02a6d..54a0b5b 100644 --- a/src/ui/activities/filetransfer/misc.rs +++ b/src/ui/activities/filetransfer/misc.rs @@ -147,8 +147,10 @@ impl FileTransferActivity { } ProtocolParams::AwsS3(params) => { info!( - "Client is not connected to remote; connecting to {} ({})", - params.bucket_name, params.region + "Client is not connected to remote; connecting to {}{} ({})", + params.endpoint.as_deref().unwrap_or(""), + params.bucket_name, + params.region.as_deref().unwrap_or("custom") ); format!("Connecting to {}…", params.bucket_name) } diff --git a/src/utils/parser.rs b/src/utils/parser.rs index aaa1de1..25e2f1d 100644 --- a/src/utils/parser.rs +++ b/src/utils/parser.rs @@ -243,7 +243,7 @@ fn parse_s3_remote_opt(s: &str) -> Result { groups.get(4).map(|group| PathBuf::from(group.as_str())); Ok(FileTransferParams::new( FileTransferProtocol::AwsS3, - ProtocolParams::AwsS3(AwsS3Params::new(bucket, region, profile)), + ProtocolParams::AwsS3(AwsS3Params::new(bucket, Some(region), profile)), ) .entry_directory(entry_directory)) } @@ -500,7 +500,7 @@ mod tests { assert_eq!(result.protocol, FileTransferProtocol::AwsS3); assert_eq!(result.entry_directory, None); assert_eq!(params.bucket_name.as_str(), "mybucket"); - assert_eq!(params.region.as_str(), "eu-central-1"); + assert_eq!(params.region.as_deref().unwrap(), "eu-central-1"); assert_eq!(params.profile, None); // With profile let result: FileTransferParams = @@ -511,7 +511,7 @@ mod tests { assert_eq!(result.protocol, FileTransferProtocol::AwsS3); assert_eq!(result.entry_directory, None); assert_eq!(params.bucket_name.as_str(), "mybucket"); - assert_eq!(params.region.as_str(), "eu-central-1"); + assert_eq!(params.region.as_deref().unwrap(), "eu-central-1"); assert_eq!(params.profile.as_deref(), Some("default")); // With wrkdir only let result: FileTransferParams = @@ -522,7 +522,7 @@ mod tests { assert_eq!(result.protocol, FileTransferProtocol::AwsS3); assert_eq!(result.entry_directory, Some(PathBuf::from("/foobar"))); assert_eq!(params.bucket_name.as_str(), "mybucket"); - assert_eq!(params.region.as_str(), "eu-central-1"); + assert_eq!(params.region.as_deref().unwrap(), "eu-central-1"); assert_eq!(params.profile, None); // With all arguments let result: FileTransferParams = @@ -533,7 +533,7 @@ mod tests { assert_eq!(result.protocol, FileTransferProtocol::AwsS3); assert_eq!(result.entry_directory, Some(PathBuf::from("/foobar"))); assert_eq!(params.bucket_name.as_str(), "mybucket"); - assert_eq!(params.region.as_str(), "eu-central-1"); + assert_eq!(params.region.as_deref().unwrap(), "eu-central-1"); assert_eq!(params.profile.as_deref(), Some("default")); // -- bad args assert!(parse_remote_opt(&String::from("s3://mybucket:default:/foobar")).is_err());