Initial Commit
commit
d5849b7203
GPG Key ID:
AF241B0AA237BBA2
|
|
@ -0,0 +1 @@
|
|||
/target
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,14 @@
|
|||
[package]
|
||||
name = "cdn-upload"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
|
||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
|
||||
[dependencies]
|
||||
rust-s3 = "0.31.0"
|
||||
webp = "0.2.2"
|
||||
image = "0.24.2"
|
||||
tokio = { version = "1.0", features = ["full"] }
|
||||
chrono = "0.4.19"
|
||||
clap = "3.1.18"
|
||||
|
|
@ -0,0 +1,72 @@
|
|||
use clap::Command;
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() {
|
||||
let bucket = clap::Arg::new("bucket")
|
||||
.help("The bucket to upload to")
|
||||
.long("bucket")
|
||||
.short('b')
|
||||
.takes_value(true)
|
||||
.required(true);
|
||||
let access_key = clap::Arg::new("access-key")
|
||||
.help("The access key to use")
|
||||
.long("access-key")
|
||||
.short('a')
|
||||
.takes_value(true)
|
||||
.required(true);
|
||||
let secret_key = clap::Arg::new("secret-key")
|
||||
.help("The secret key to use")
|
||||
.long("secret-key")
|
||||
.short('s')
|
||||
.takes_value(true)
|
||||
.required(true);
|
||||
let region = clap::Arg::new("region")
|
||||
.help("The region to use")
|
||||
.long("region")
|
||||
.short('r')
|
||||
.takes_value(true)
|
||||
.default_value("us-east-1");
|
||||
let endpoint = clap::Arg::new("endpoint")
|
||||
.help("The endpoint to use")
|
||||
.long("endpoint")
|
||||
.short('e')
|
||||
.takes_value(true);
|
||||
let input = clap::Arg::new("input")
|
||||
.help("The input directory")
|
||||
.index(1)
|
||||
.multiple_values(true);
|
||||
|
||||
let matches = Command::new("S3 Image Uploader")
|
||||
.version("1.0")
|
||||
.author("Julius")
|
||||
.about("used to upload an image to s3")
|
||||
.args(&[bucket, access_key, secret_key, region, endpoint, input])
|
||||
.get_matches();
|
||||
|
||||
let u = cdn_upload::Uploader::new(
|
||||
matches.value_of("bucket").unwrap(),
|
||||
matches.value_of("access-key").unwrap(),
|
||||
matches.value_of("secret-key").unwrap(),
|
||||
matches.value_of("region").unwrap(),
|
||||
matches.value_of("endpoint"),
|
||||
)
|
||||
.unwrap();
|
||||
let r = u.existing_files().await.unwrap();
|
||||
let m = matches.values_of("input").unwrap().map(ToString::to_string);
|
||||
let handles = m.map(|arg| {
|
||||
let u = u.clone();
|
||||
let r = r.clone();
|
||||
tokio::spawn(async move {
|
||||
let filename = arg;
|
||||
let data = cdn_upload::fix_image(filename.clone(), &r).unwrap();
|
||||
for img in data {
|
||||
u.upload(&img).await.unwrap();
|
||||
println!("Uploaded {} as {}", filename, img.name());
|
||||
}
|
||||
})
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
for handle in handles {
|
||||
handle.await.unwrap();
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,258 @@
|
|||
use std::{io::Cursor, ops::Deref, path::Path, str::FromStr};
|
||||
|
||||
use s3::Bucket;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum Error {
|
||||
ImageError(image::ImageError),
|
||||
WebpError(String),
|
||||
IoError(std::io::Error),
|
||||
CredentialsError(s3::creds::error::CredentialsError),
|
||||
S3Error(s3::error::S3Error),
|
||||
UploadError(String),
|
||||
ParseError(String),
|
||||
ParseIntError(std::num::ParseIntError),
|
||||
}
|
||||
|
||||
impl From<image::ImageError> for Error {
|
||||
fn from(err: image::ImageError) -> Error {
|
||||
Error::ImageError(err)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<std::io::Error> for Error {
|
||||
fn from(err: std::io::Error) -> Error {
|
||||
Error::IoError(err)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<s3::creds::error::CredentialsError> for Error {
|
||||
fn from(err: s3::creds::error::CredentialsError) -> Error {
|
||||
Error::CredentialsError(err)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<s3::error::S3Error> for Error {
|
||||
fn from(err: s3::error::S3Error) -> Error {
|
||||
Error::S3Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<std::num::ParseIntError> for Error {
|
||||
fn from(err: std::num::ParseIntError) -> Error {
|
||||
Error::ParseIntError(err)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct Image {
|
||||
image_type: ImageType,
|
||||
name: String,
|
||||
width: u32,
|
||||
data: Vec<u8>,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for Image {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("Image")
|
||||
.field("name", &self.name)
|
||||
.field("width", &self.width)
|
||||
.field("image_type", &self.image_type)
|
||||
.finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
|
||||
impl Image {
|
||||
pub fn name(&self) -> String {
|
||||
if self.width == 0 {
|
||||
format!("{}.{}", self.name, self.image_type.extension())
|
||||
} else {
|
||||
format!(
|
||||
"{}-{}.{}",
|
||||
self.name,
|
||||
self.width,
|
||||
self.image_type.extension()
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl FromStr for Image {
|
||||
type Err = Error;
|
||||
|
||||
fn from_str(s: &str) -> Result<Self, Self::Err> {
|
||||
let mut parts = s.split('.');
|
||||
let name = parts.next().unwrap().to_owned();
|
||||
let image_type = parts.next().unwrap().parse()?;
|
||||
let r = name.split(|c| c == '-').collect::<Vec<_>>();
|
||||
let name = r[0].to_owned();
|
||||
let width = if r.len() > 1 { r[1].parse()? } else { 0 };
|
||||
Ok(Image {
|
||||
image_type,
|
||||
name,
|
||||
width,
|
||||
data: vec![],
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
|
||||
pub enum ImageType {
|
||||
Jpg,
|
||||
Webp,
|
||||
}
|
||||
|
||||
impl FromStr for ImageType {
|
||||
type Err = Error;
|
||||
|
||||
fn from_str(s: &str) -> Result<Self, Self::Err> {
|
||||
match s {
|
||||
"jpg" => Ok(ImageType::Jpg),
|
||||
"webp" => Ok(ImageType::Webp),
|
||||
_ => Err(Error::ParseError(s.to_owned())),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ImageType {
|
||||
pub fn extension(&self) -> &str {
|
||||
match self {
|
||||
ImageType::Jpg => "jpg",
|
||||
ImageType::Webp => "webp",
|
||||
}
|
||||
}
|
||||
|
||||
pub fn content_type(&self) -> &str {
|
||||
match self {
|
||||
ImageType::Jpg => "image/jpeg",
|
||||
ImageType::Webp => "image/webp",
|
||||
}
|
||||
}
|
||||
|
||||
pub fn options() -> Vec<ImageType> {
|
||||
vec![ImageType::Jpg, ImageType::Webp]
|
||||
}
|
||||
}
|
||||
|
||||
impl Deref for Image {
|
||||
type Target = [u8];
|
||||
fn deref(&self) -> &[u8] {
|
||||
&self.data
|
||||
}
|
||||
}
|
||||
|
||||
pub type Images = Vec<Image>;
|
||||
|
||||
pub const SIZES: [u32; 5] = [0, 150, 200, 300, 450];
|
||||
|
||||
pub fn fix_image(filename: impl AsRef<Path>, existing_images: &[Image]) -> Result<Images, Error> {
|
||||
let img = image::open(filename.as_ref())?;
|
||||
let z = filename.as_ref().file_name();
|
||||
let name = z.unwrap().to_str().unwrap().to_string();
|
||||
let name = name.split(|c| c == '.').nth(0).unwrap().to_string();
|
||||
|
||||
let mut v = vec![];
|
||||
for image_type in ImageType::options() {
|
||||
for width in SIZES {
|
||||
if existing_images
|
||||
.iter()
|
||||
.any(|img| img.name == name && img.width == width && img.image_type == image_type)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
let img = img.resize(
|
||||
if width == 0 { 450 } else { width },
|
||||
999999,
|
||||
image::imageops::Lanczos3,
|
||||
);
|
||||
let img = image::DynamicImage::ImageRgba8(img.to_rgba8());
|
||||
let data = match image_type {
|
||||
ImageType::Jpg => {
|
||||
let mut jpg_buf = vec![];
|
||||
img.write_to(
|
||||
&mut Cursor::new(&mut jpg_buf),
|
||||
image::ImageOutputFormat::Jpeg(75),
|
||||
)?;
|
||||
jpg_buf
|
||||
}
|
||||
ImageType::Webp => {
|
||||
let z = webp::Encoder::from_image(&img)
|
||||
.map_err(|e| Error::WebpError(e.to_string()))?;
|
||||
let d = z.encode(50.0);
|
||||
d.to_vec()
|
||||
}
|
||||
};
|
||||
|
||||
v.push(Image {
|
||||
image_type,
|
||||
name: name.clone(),
|
||||
width,
|
||||
data,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
Ok(v)
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct Uploader {
|
||||
bucket: Bucket,
|
||||
date: String,
|
||||
}
|
||||
|
||||
impl Uploader {
|
||||
pub fn new(
|
||||
bucket: &str,
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
region: &str,
|
||||
endpoint: Option<&str>,
|
||||
) -> Result<Uploader, Error> {
|
||||
let mut region = s3::Region::from_str(region).unwrap();
|
||||
if let Some(endpoint) = endpoint {
|
||||
region = s3::Region::Custom {
|
||||
endpoint: endpoint.to_string(),
|
||||
region: format!("{}", region),
|
||||
};
|
||||
}
|
||||
let bucket = Bucket::new(
|
||||
bucket,
|
||||
region,
|
||||
s3::creds::Credentials::new(Some(access_key), Some(secret_key), None, None, None)?,
|
||||
)?
|
||||
.with_path_style();
|
||||
Ok(Uploader {
|
||||
bucket,
|
||||
date: chrono::Local::now().date().format("%Y%m%d").to_string(),
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn upload(&self, data: &Image) -> Result<String, Error> {
|
||||
let filename = format!("{}/{}", self.date, data.name());
|
||||
let (data, code) = self
|
||||
.bucket
|
||||
.put_object_with_content_type(&filename, &data, data.image_type.content_type())
|
||||
.await?;
|
||||
// println!("{}", );
|
||||
if code != 200 {
|
||||
return Err(Error::UploadError(
|
||||
data.iter().map(|d| char::from(*d)).collect::<String>(),
|
||||
));
|
||||
}
|
||||
Ok(filename)
|
||||
}
|
||||
|
||||
pub async fn existing_files(&self) -> Result<Vec<Image>, Error> {
|
||||
let z = self.bucket.list(format!("/{}", self.date), None).await?;
|
||||
let mut all_files = vec![];
|
||||
for w in z {
|
||||
for x in w.contents {
|
||||
let name = &x.key;
|
||||
all_files.push(name.split_at(self.date.len() + 1).1.to_string().parse()?)
|
||||
}
|
||||
}
|
||||
|
||||
Ok(all_files)
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue