Merge branch 'main' of https://github.com/LemmyNet/lemmy into ui_settings

This commit is contained in:
Domenic Horner 2023-07-03 11:45:12 +08:00
commit 64e93aac3a
28 changed files with 185 additions and 72 deletions

2
.gitattributes vendored Normal file
View file

@ -0,0 +1,2 @@
# Normalize EOL for all files that Git considers text files.
* text=auto eol=lf

4
.gitignore vendored
View file

@ -26,3 +26,7 @@ pictrs/
# The generated typescript bindings
bindings
# Database cluster and sockets for testing
dev_pgdata/
*.PGSQL.*

View file

@ -60,6 +60,9 @@ pipeline:
-D clippy::unused_self
-A clippy::uninlined_format_args
-D clippy::get_first
-D clippy::explicit_into_iter_loop
-D clippy::explicit_iter_loop
-D clippy::needless_collect
- cargo clippy --workspace --features console --
-D clippy::unwrap_used
-D clippy::indexing_slicing

45
Cargo.lock generated
View file

@ -14,9 +14,9 @@ dependencies = [
[[package]]
name = "activitypub_federation"
version = "0.4.4"
version = "0.4.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "27540f6c4b72c91176610ed5279061a021387f972c7c6f42c41032b78a808267"
checksum = "4ab3ac148d9c0b4163a6d41040c17de7558a42224b9ecbd4e8f033aef6c254d9"
dependencies = [
"activitystreams-kinds",
"actix-web",
@ -399,6 +399,9 @@ name = "anyhow"
version = "1.0.71"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8"
dependencies = [
"backtrace",
]
[[package]]
name = "argparse"
@ -2570,7 +2573,7 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
[[package]]
name = "lemmy_api"
version = "0.18.1-rc.1"
version = "0.18.1-rc.4"
dependencies = [
"actix-web",
"anyhow",
@ -2590,11 +2593,12 @@ dependencies = [
"tokio",
"tracing",
"uuid",
"wav",
]
[[package]]
name = "lemmy_api_common"
version = "0.18.1-rc.1"
version = "0.18.1-rc.4"
dependencies = [
"actix-web",
"anyhow",
@ -2623,7 +2627,7 @@ dependencies = [
[[package]]
name = "lemmy_api_crud"
version = "0.18.1-rc.1"
version = "0.18.1-rc.4"
dependencies = [
"activitypub_federation",
"actix-web",
@ -2644,7 +2648,7 @@ dependencies = [
[[package]]
name = "lemmy_apub"
version = "0.18.1-rc.1"
version = "0.18.1-rc.4"
dependencies = [
"activitypub_federation",
"actix-web",
@ -2682,7 +2686,7 @@ dependencies = [
[[package]]
name = "lemmy_db_schema"
version = "0.18.1-rc.1"
version = "0.18.1-rc.4"
dependencies = [
"activitypub_federation",
"async-trait",
@ -2719,7 +2723,7 @@ dependencies = [
[[package]]
name = "lemmy_db_views"
version = "0.18.1-rc.1"
version = "0.18.1-rc.4"
dependencies = [
"diesel",
"diesel-async",
@ -2736,7 +2740,7 @@ dependencies = [
[[package]]
name = "lemmy_db_views_actor"
version = "0.18.1-rc.1"
version = "0.18.1-rc.4"
dependencies = [
"diesel",
"diesel-async",
@ -2749,7 +2753,7 @@ dependencies = [
[[package]]
name = "lemmy_db_views_moderator"
version = "0.18.1-rc.1"
version = "0.18.1-rc.4"
dependencies = [
"diesel",
"diesel-async",
@ -2761,7 +2765,7 @@ dependencies = [
[[package]]
name = "lemmy_routes"
version = "0.18.1-rc.1"
version = "0.18.1-rc.4"
dependencies = [
"activitypub_federation",
"actix-web",
@ -2786,7 +2790,7 @@ dependencies = [
[[package]]
name = "lemmy_server"
version = "0.18.1-rc.1"
version = "0.18.1-rc.4"
dependencies = [
"activitypub_federation",
"actix-cors",
@ -2828,7 +2832,7 @@ dependencies = [
[[package]]
name = "lemmy_utils"
version = "0.18.1-rc.1"
version = "0.18.1-rc.4"
dependencies = [
"actix-web",
"anyhow",
@ -4445,6 +4449,12 @@ dependencies = [
"bytemuck",
]
[[package]]
name = "riff"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b9b1a3d5f46d53f4a3478e2be4a5a5ce5108ea58b100dcd139830eae7f79a3a1"
[[package]]
name = "ring"
version = "0.16.20"
@ -6256,6 +6266,15 @@ dependencies = [
"web-sys",
]
[[package]]
name = "wav"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a65e199c799848b4f997072aa4d673c034f80f40191f97fe2f0a23f410be1609"
dependencies = [
"riff",
]
[[package]]
name = "web-sys"
version = "0.3.60"

View file

@ -1,5 +1,5 @@
[workspace.package]
version = "0.18.1-rc.1"
version = "0.18.1-rc.4"
edition = "2021"
description = "A link aggregator for the fediverse"
license = "AGPL-3.0"
@ -21,14 +21,9 @@ repository.workspace = true
doctest = false
[profile.release]
strip = "symbols"
debug = 0
lto = "thin"
[profile.dev]
strip = "symbols"
debug = 0
[features]
embed-pictrs = ["pict-rs"]
console = ["console-subscriber", "opentelemetry", "opentelemetry-otlp", "tracing-opentelemetry", "reqwest-tracing/opentelemetry_0_16"]
@ -49,16 +44,16 @@ members = [
]
[workspace.dependencies]
lemmy_api = { version = "=0.18.1-rc.1", path = "./crates/api" }
lemmy_api_crud = { version = "=0.18.1-rc.1", path = "./crates/api_crud" }
lemmy_apub = { version = "=0.18.1-rc.1", path = "./crates/apub" }
lemmy_utils = { version = "=0.18.1-rc.1", path = "./crates/utils" }
lemmy_db_schema = { version = "=0.18.1-rc.1", path = "./crates/db_schema" }
lemmy_api_common = { version = "=0.18.1-rc.1", path = "./crates/api_common" }
lemmy_routes = { version = "=0.18.1-rc.1", path = "./crates/routes" }
lemmy_db_views = { version = "=0.18.1-rc.1", path = "./crates/db_views" }
lemmy_db_views_actor = { version = "=0.18.1-rc.1", path = "./crates/db_views_actor" }
lemmy_db_views_moderator = { version = "=0.18.1-rc.1", path = "./crates/db_views_moderator" }
lemmy_api = { version = "=0.18.1-rc.4", path = "./crates/api" }
lemmy_api_crud = { version = "=0.18.1-rc.4", path = "./crates/api_crud" }
lemmy_apub = { version = "=0.18.1-rc.4", path = "./crates/apub" }
lemmy_utils = { version = "=0.18.1-rc.4", path = "./crates/utils" }
lemmy_db_schema = { version = "=0.18.1-rc.4", path = "./crates/db_schema" }
lemmy_api_common = { version = "=0.18.1-rc.4", path = "./crates/api_common" }
lemmy_routes = { version = "=0.18.1-rc.4", path = "./crates/routes" }
lemmy_db_views = { version = "=0.18.1-rc.4", path = "./crates/db_views" }
lemmy_db_views_actor = { version = "=0.18.1-rc.4", path = "./crates/db_views_actor" }
lemmy_db_views_moderator = { version = "=0.18.1-rc.4", path = "./crates/db_views_moderator" }
activitypub_federation = { version = "0.4.4", default-features = false, features = ["actix-web"] }
diesel = "2.1.0"
diesel_migrations = "2.1.0"
@ -85,7 +80,7 @@ base64 = "0.13.1"
uuid = { version = "1.3.4", features = ["serde", "v4"] }
async-trait = "0.1.68"
captcha = "0.0.9"
anyhow = "1.0.71"
anyhow = { version = "1.0.71", features = ["backtrace"] } # backtrace is on by default on nightly, but not stable rust
diesel_ltree = "0.3.0"
typed-builder = "0.10.0"
serial_test = "0.9.0"

View file

@ -30,6 +30,7 @@ captcha = { workspace = true }
anyhow = { workspace = true }
tracing = { workspace = true }
chrono = { workspace = true }
wav = "1.0.0"
[dev-dependencies]
serial_test = { workspace = true }

View file

@ -3,6 +3,7 @@ use captcha::Captcha;
use lemmy_api_common::{context::LemmyContext, utils::local_site_to_slur_regex};
use lemmy_db_schema::source::local_site::LocalSite;
use lemmy_utils::{error::LemmyError, utils::slurs::check_slurs};
use std::io::Cursor;
mod comment;
mod comment_report;
@ -22,18 +23,42 @@ pub trait Perform {
}
/// Converts the captcha to a base64 encoded wav audio file
pub(crate) fn captcha_as_wav_base64(captcha: &Captcha) -> String {
pub(crate) fn captcha_as_wav_base64(captcha: &Captcha) -> Result<String, LemmyError> {
let letters = captcha.as_wav();
let mut concat_letters: Vec<u8> = Vec::new();
// Decode each wav file, concatenate the samples
let mut concat_samples: Vec<i16> = Vec::new();
let mut any_header: Option<wav::Header> = None;
for letter in letters {
let bytes = letter.unwrap_or_default();
concat_letters.extend(bytes);
let mut cursor = Cursor::new(letter.unwrap_or_default());
let (header, samples) = wav::read(&mut cursor)?;
any_header = Some(header);
if let Some(samples16) = samples.as_sixteen() {
concat_samples.extend(samples16);
} else {
return Err(LemmyError::from_message("couldnt_create_audio_captcha"));
}
}
// Convert to base64
base64::encode(concat_letters)
// Encode the concatenated result as a wav file
let mut output_buffer = Cursor::new(vec![]);
let header = match any_header {
Some(header) => header,
None => return Err(LemmyError::from_message("couldnt_create_audio_captcha")),
};
let wav_write_result = wav::write(
header,
&wav::BitDepth::Sixteen(concat_samples),
&mut output_buffer,
);
if let Err(e) = wav_write_result {
return Err(LemmyError::from_error_message(
e,
"couldnt_create_audio_captcha",
));
}
Ok(base64::encode(output_buffer.into_inner()))
}
/// Check size of report and remove whitespace

View file

@ -33,7 +33,7 @@ impl Perform for GetCaptcha {
let png = captcha.as_base64().expect("failed to generate captcha");
let wav = captcha_as_wav_base64(&captcha);
let wav = captcha_as_wav_base64(&captcha)?;
let captcha_form: CaptchaAnswerForm = CaptchaAnswerForm { answer };
// Stores the captcha item in the db

View file

@ -24,3 +24,10 @@ As you can see, each API endpoint needs a parameter type ( GetPosts), path (/pos
For a real example of a Lemmy API client, look at [lemmyBB](https://github.com/LemmyNet/lemmyBB/tree/main/src/api).
Lemmy also provides a websocket API. You can find the full websocket code in [this file](https://github.com/LemmyNet/lemmy/blob/main/src/api_routes_websocket.rs).
## Generate TypeScript bindings
TypeScript bindings (API types) can be generated by running `cargo test --features full`.
The ts files be generated into a `bindings` folder.
This crate uses [`ts_rs`](https://docs.rs/ts-rs/6.2.1/ts_rs/#traits) macros `derive(TS)` and `ts(export)` to attribute types for binding generating.

View file

@ -98,7 +98,6 @@ pub async fn send_local_notifs(
for mention in mentions
.iter()
.filter(|m| m.is_local(&context.settings().hostname) && m.name.ne(&person.name))
.collect::<Vec<&MentionData>>()
{
let mention_name = mention.name.clone();
let user_view = LocalUserView::read_from_name(context.pool(), &mention_name).await;

View file

@ -193,6 +193,7 @@ pub async fn fetch_site_data(
client: &ClientWithMiddleware,
settings: &Settings,
url: Option<&Url>,
include_image: bool,
) -> (Option<SiteMetadata>, Option<DbUrl>) {
match &url {
Some(url) => {
@ -200,6 +201,9 @@ pub async fn fetch_site_data(
// Ignore errors, since it may be an image, or not have the data.
// Warning, this may ignore SSL errors
let metadata_option = fetch_site_metadata(client, url).await.ok();
if !include_image {
return (metadata_option, None);
}
let missing_pictrs_file =
|r: PictrsResponse| r.files.first().expect("missing pictrs file").file.clone();

View file

@ -428,6 +428,13 @@ pub fn local_site_opt_to_slur_regex(local_site: &Option<LocalSite>) -> Option<Re
.unwrap_or(None)
}
pub fn local_site_opt_to_sensitive(local_site: &Option<LocalSite>) -> bool {
local_site
.as_ref()
.map(|site| site.enable_nsfw)
.unwrap_or(false)
}
pub fn send_application_approved_email(
user: &LocalUserView,
settings: &Settings,

View file

@ -22,5 +22,5 @@ tracing = { workspace = true }
url = { workspace = true }
async-trait = { workspace = true }
webmention = "0.4.0"
chrono = { worspace = true }
chrono = { workspace = true }
uuid = { workspace = true }

View file

@ -191,7 +191,7 @@ impl PerformCrud for CreateComment {
pub fn check_comment_depth(comment: &Comment) -> Result<(), LemmyError> {
let path = &comment.path.0;
let length = path.split('.').collect::<Vec<&str>>().len();
let length = path.split('.').count();
if length > MAX_COMMENT_DEPTH_LIMIT {
Err(LemmyError::from_message("max_comment_depth_reached"))
} else {

View file

@ -79,7 +79,7 @@ impl PerformCrud for CreatePost {
// Fetch post links and pictrs cached image
let (metadata_res, thumbnail_url) =
fetch_site_data(context.client(), context.settings(), data_url).await;
fetch_site_data(context.client(), context.settings(), data_url, true).await;
let (embed_title, embed_description, embed_video_url) = metadata_res
.map(|u| (u.title, u.description, u.embed_video_url))
.unwrap_or_default();

View file

@ -69,7 +69,7 @@ impl PerformCrud for EditPost {
// Fetch post links and Pictrs cached image
let data_url = data.url.as_ref();
let (metadata_res, thumbnail_url) =
fetch_site_data(context.client(), context.settings(), data_url).await;
fetch_site_data(context.client(), context.settings(), data_url, true).await;
let (embed_title, embed_description, embed_video_url) = metadata_res
.map(|u| (Some(u.title), Some(u.description), Some(u.embed_video_url)))
.unwrap_or_default();

View file

@ -43,12 +43,11 @@ pub(crate) async fn send_activity_in_community(
// send to user followers
if !is_mod_action {
inboxes.append(
inboxes.extend(
&mut PersonFollower::list_followers(context.pool(), actor.id)
.await?
.into_iter()
.map(|p| ApubPerson(p).shared_inbox_or_inbox())
.collect(),
.map(|p| ApubPerson(p).shared_inbox_or_inbox()),
);
}

View file

@ -11,10 +11,7 @@ use lemmy_db_schema::{
traits::Crud,
utils::DbPool,
};
use lemmy_utils::{
error::LemmyError,
utils::mention::{scrape_text_for_mentions, MentionData},
};
use lemmy_utils::{error::LemmyError, utils::mention::scrape_text_for_mentions};
use serde::{Deserialize, Serialize};
use serde_json::Value;
use url::Url;
@ -67,10 +64,9 @@ pub async fn collect_non_local_mentions(
let mentions = scrape_text_for_mentions(&comment.content)
.into_iter()
// Filter only the non-local ones
.filter(|m| !m.is_local(&context.settings().hostname))
.collect::<Vec<MentionData>>();
.filter(|m| !m.is_local(&context.settings().hostname));
for mention in &mentions {
for mention in mentions {
let identifier = format!("{}@{}", mention.name, mention.domain);
let person = webfinger_resolve_actor::<LemmyContext, ApubPerson>(&identifier, context).await;
if let Ok(person) = person {

View file

@ -25,7 +25,7 @@ use html2md::parse_html;
use lemmy_api_common::{
context::LemmyContext,
request::fetch_site_data,
utils::{is_mod_or_admin, local_site_opt_to_slur_regex},
utils::{is_mod_or_admin, local_site_opt_to_sensitive, local_site_opt_to_slur_regex},
};
use lemmy_db_schema::{
self,
@ -197,18 +197,33 @@ impl Object for ApubPost {
} else {
None
};
let local_site = LocalSite::read(context.pool()).await.ok();
let allow_sensitive = local_site_opt_to_sensitive(&local_site);
let page_is_sensitive = page.sensitive.unwrap_or(false);
let include_image = allow_sensitive || !page_is_sensitive;
// Only fetch metadata if the post has a url and was not seen previously. We dont want to
// waste resources by fetching metadata for the same post multiple times.
let (metadata_res, thumbnail_url) = match &url {
// Additionally, only fetch image if content is not sensitive or is allowed on local site.
let (metadata_res, thumbnail) = match &url {
Some(url) if old_post.is_err() => {
fetch_site_data(context.client(), context.settings(), Some(url)).await
fetch_site_data(
context.client(),
context.settings(),
Some(url),
include_image,
)
.await
}
_ => (None, page.image.map(|i| i.url.into())),
_ => (None, None),
};
// If no image was included with metadata, use post image instead when available.
let thumbnail_url = thumbnail.or_else(|| page.image.map(|i| i.url.into()));
let (embed_title, embed_description, embed_video_url) = metadata_res
.map(|u| (u.title, u.description, u.embed_video_url))
.unwrap_or_default();
let local_site = LocalSite::read(context.pool()).await.ok();
let slur_regex = &local_site_opt_to_slur_regex(&local_site);
let body_slurs_removed =

View file

@ -48,7 +48,7 @@ rustls = { workspace = true }
futures-util = { workspace = true }
tokio-postgres = { workspace = true }
tokio-postgres-rustls = { workspace = true }
uuid = { features = ["v4"] }
uuid = { workspace = true, features = ["v4"] }
[dev-dependencies]
serial_test = { workspace = true }

View file

@ -99,8 +99,7 @@ impl Comment {
// left join comment c2 on c2.path <@ c.path and c2.path != c.path
// group by c.id
let path_split = parent_path.0.split('.').collect::<Vec<&str>>();
let parent_id = path_split.get(1);
let parent_id = parent_path.0.split('.').nth(1);
if let Some(parent_id) = parent_id {
let top_parent = format!("0.{}", parent_id);

View file

@ -90,7 +90,11 @@ impl Display for LemmyError {
if let Some(message) = &self.message {
write!(f, "{message}: ")?;
}
writeln!(f, "{}", self.inner)?;
// print anyhow including trace
// https://docs.rs/anyhow/latest/anyhow/struct.Error.html#display-representations
// this will print the anyhow trace (only if it exists)
// and if RUST_BACKTRACE=1, also a full backtrace
writeln!(f, "{:?}", self.inner)?;
fmt::Display::fmt(&self.context, f)
}
}

@ -1 +1 @@
Subproject commit b3cca4b7e26dd7d9a389f75c6e50a489d93791ea
Subproject commit 5a9d44656e2658ab7cb2dbec3fd1bfaf57654533

View file

@ -4,7 +4,7 @@ update local_user set default_sort_type = 'TopDay' where default_sort_type in ('
-- rename the old enum
alter type sort_type_enum rename to sort_type_enum__;
-- create the new enum
CREATE TYPE sort_type_enum AS ENUM ('Active', 'Hot', 'New', 'Old', 'TopDay', 'TopWeek', 'TopMonth', 'TopYear', 'TopAll', 'MostComments', 'NewComments');
CREATE TYPE sort_type_enum AS ENUM ('Active', 'Hot', 'New', 'Old', 'TopDay', 'TopWeek', 'TopMonth', 'TopYear', 'TopAll', 'MostComments', 'NewComments', 'TopHour', 'TopSixHour', 'TopTwelveHour');
-- alter all you enum columns
alter table local_user

View file

@ -14,7 +14,10 @@ cargo clippy --workspace --fix --allow-staged --allow-dirty --tests --all-target
-D clippy::manual_string_new -D clippy::redundant_closure_for_method_calls \
-D clippy::unused_self \
-A clippy::uninlined_format_args \
-D clippy::get_first
-D clippy::get_first \
-D clippy::explicit_into_iter_loop \
-D clippy::explicit_iter_loop \
-D clippy::needless_collect
cargo clippy --workspace --features console -- \
-D clippy::unwrap_used \

View file

@ -6,10 +6,14 @@ set -e
new_tag="$1"
third_semver=$(echo $new_tag | cut -d "." -f 3)
# Goto the upper route
CWD="$(cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
cd $CWD/../
# The ansible and docker installs should only update for non release-candidates
# IE, when the third semver is a number, not '2-rc'
if [ ! -z "${third_semver##*[!0-9]*}" ]; then
pushd ../docker
pushd docker
sed -i "s/dessalines\/lemmy:.*/dessalines\/lemmy:$new_tag/" docker-compose.yml
sed -i "s/dessalines\/lemmy-ui:.*/dessalines\/lemmy-ui:$new_tag/" docker-compose.yml
sed -i "s/dessalines\/lemmy-ui:.*/dessalines\/lemmy-ui:$new_tag/" federation/docker-compose.yml
@ -18,7 +22,7 @@ if [ ! -z "${third_semver##*[!0-9]*}" ]; then
popd
# Setting the version for Ansible
pushd ../../lemmy-ansible
pushd ../lemmy-ansible
echo $new_tag > "VERSION"
git add "VERSION"
git commit -m"Updating VERSION"
@ -29,14 +33,12 @@ if [ ! -z "${third_semver##*[!0-9]*}" ]; then
fi
# Update crate versions
pushd ..
old_tag=$(grep version Cargo.toml | head -1 | cut -d'"' -f 2)
sed -i "s/{ version = \"=$old_tag\", path/{ version = \"=$new_tag\", path/g" Cargo.toml
sed -i "s/version = \"$old_tag\"/version = \"$new_tag\"/g" Cargo.toml
git add Cargo.toml
cargo check
git add Cargo.lock
popd
# Update the submodules
git submodule update --remote

24
scripts/start_dev_db.sh Normal file
View file

@ -0,0 +1,24 @@
# This script is meant to be run with `source` so it can set environment variables.
export PGDATA="$PWD/dev_pgdata"
export PGHOST=$PWD
export LEMMY_DATABASE_URL="postgresql://lemmy:password@/lemmy?host=$PWD"
# If cluster exists, stop the server and delete the cluster
if [ -d $PGDATA ]
then
# Prevent `stop` from failing if server already stopped
pg_ctl restart > /dev/null
pg_ctl stop
rm -rf $PGDATA
fi
# Create cluster
initdb --username=postgres --auth=trust --no-instructions
# Start server that only listens to socket in current directory
pg_ctl start --options="-c listen_addresses= -c unix_socket_directories=$PWD" > /dev/null
# Setup database
psql -c "CREATE USER lemmy WITH PASSWORD 'password' SUPERUSER;" -U postgres
psql -c "CREATE DATABASE lemmy WITH OWNER lemmy;" -U postgres

View file

@ -1,13 +1,15 @@
#!/usr/bin/env bash
set -e
CWD="$(cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)"
cd $CWD/../
PACKAGE="$1"
echo "$PACKAGE"
psql -U lemmy -d postgres -c "DROP DATABASE lemmy;"
psql -U lemmy -d postgres -c "CREATE DATABASE lemmy;"
source scripts/start_dev_db.sh
export LEMMY_DATABASE_URL=postgres://lemmy:password@localhost:5432/lemmy
# tests are executed in working directory crates/api (or similar),
# so to load the config we need to traverse to the repo root
export LEMMY_CONFIG_LOCATION=../../config/config.hjson
@ -21,3 +23,6 @@ else
fi
# Add this to do printlns: -- --nocapture
pg_ctl stop
rm -rf $PGDATA