From be10ad46630138697fa55ad85a8370796cb99731 Mon Sep 17 00:00:00 2001 From: "byte[]" Date: Mon, 6 Jul 2020 14:00:02 -0400 Subject: [PATCH 01/13] add computed duration field --- lib/philomena/elasticsearch.ex | 2 +- lib/philomena/images/elasticsearch_index.ex | 6 ++++++ lib/philomena/images/image.ex | 3 +++ lib/philomena/images/query.ex | 6 +++--- lib/philomena/uploader.ex | 1 + lib/philomena_web/image_sorter.ex | 3 +++ .../templates/image/_image_meta.html.slime | 8 ++++++++ lib/philomena_web/templates/image/new.html.slime | 1 + lib/philomena_web/views/api/json/image_view.ex | 2 ++ .../20200706171350_add_duration_to_images.exs | 14 ++++++++++++++ priv/repo/structure.sql | 4 +++- 11 files changed, 45 insertions(+), 5 deletions(-) create mode 100644 priv/repo/migrations/20200706171350_add_duration_to_images.exs diff --git a/lib/philomena/elasticsearch.ex b/lib/philomena/elasticsearch.ex index 4edd0c3b6..9374ce472 100644 --- a/lib/philomena/elasticsearch.ex +++ b/lib/philomena/elasticsearch.ex @@ -52,7 +52,7 @@ defmodule Philomena.Elasticsearch do index_name = index.index_name() mapping = index.mapping().mappings.properties - Elastix.Mapping.put(elastic_url(), index_name, "_doc", %{properties: mapping}) + Elastix.Mapping.put(elastic_url(), index_name, "_doc", %{properties: mapping}, include_type_name: true) end def index_document(doc, module) do diff --git a/lib/philomena/images/elasticsearch_index.ex b/lib/philomena/images/elasticsearch_index.ex index 3e94e1a0f..193bab3d7 100644 --- a/lib/philomena/images/elasticsearch_index.ex +++ b/lib/philomena/images/elasticsearch_index.ex @@ -31,6 +31,7 @@ defmodule Philomena.Images.ElasticsearchIndex do downvoters: %{type: "keyword"}, downvotes: %{type: "integer"}, duplicate_id: %{type: "integer"}, + duration: %{type: "float"}, faves: %{type: "integer"}, favourited_by_user_ids: %{type: "keyword"}, favourited_by_users: %{type: "keyword"}, @@ -46,7 +47,9 @@ defmodule Philomena.Images.ElasticsearchIndex do mime_type: %{type: "keyword"}, orig_sha512_hash: %{type: "keyword"}, original_format: %{type: "keyword"}, + pixels: %{type: "integer"}, score: %{type: "integer"}, + size: %{type: "integer"}, sha512_hash: %{type: "keyword"}, source_url: %{type: "keyword"}, tag_count: %{type: "integer"}, @@ -93,6 +96,9 @@ defmodule Philomena.Images.ElasticsearchIndex do comment_count: image.comments_count, width: image.image_width, height: image.image_height, + pixels: image.image_width * image.image_height, + size: image.image_size, + duration: image.image_duration, tag_count: length(image.tags), aspect_ratio: image.image_aspect_ratio, wilson_score: wilson_score(image), diff --git a/lib/philomena/images/image.ex b/lib/philomena/images/image.ex index 3868ae240..ab7f25b2a 100644 --- a/lib/philomena/images/image.ex +++ b/lib/philomena/images/image.ex @@ -49,6 +49,7 @@ defmodule Philomena.Images.Image do field :image_format, :string field :image_mime_type, :string field :image_aspect_ratio, :float + field :image_duration, :float field :image_is_animated, :boolean, source: :is_animated field :ip, EctoNetwork.INET field :fingerprint, :string @@ -131,6 +132,7 @@ defmodule Philomena.Images.Image do :image_format, :image_mime_type, :image_aspect_ratio, + :image_duration, :image_orig_sha512_hash, :image_sha512_hash, :uploaded_image, @@ -145,6 +147,7 @@ defmodule Philomena.Images.Image do :image_format, :image_mime_type, :image_aspect_ratio, + :image_duration, :image_orig_sha512_hash, :image_sha512_hash, :uploaded_image, diff --git a/lib/philomena/images/query.ex b/lib/philomena/images/query.ex index faec17cc2..d7af3b0f2 100644 --- a/lib/philomena/images/query.ex +++ b/lib/philomena/images/query.ex @@ -69,11 +69,11 @@ defmodule Philomena.Images.Query do defp anonymous_fields do [ int_fields: - ~W(id width height comment_count score upvotes downvotes faves uploader_id faved_by_id tag_count), - float_fields: ~W(aspect_ratio wilson_score), + ~W(id width height comment_count score upvotes downvotes faves uploader_id faved_by_id tag_count pixels size), + float_fields: ~W(aspect_ratio wilson_score duration), date_fields: ~W(created_at updated_at first_seen_at), literal_fields: - ~W(faved_by orig_sha512_hash sha512_hash uploader source_url original_format), + ~W(faved_by orig_sha512_hash sha512_hash uploader source_url original_format mime_type), ngram_fields: ~W(description), custom_fields: ~W(gallery_id), default_field: {"namespaced_tags.name", :term}, diff --git a/lib/philomena/uploader.ex b/lib/philomena/uploader.ex index c246f1029..ba2cb0f42 100644 --- a/lib/philomena/uploader.ex +++ b/lib/philomena/uploader.ex @@ -31,6 +31,7 @@ defmodule Philomena.Uploader do "size" => analysis.size, "format" => analysis.extension, "mime_type" => analysis.mime_type, + "duration" => analysis.duration, "aspect_ratio" => analysis.aspect_ratio, "orig_sha512_hash" => analysis.sha512, "sha512_hash" => analysis.sha512, diff --git a/lib/philomena_web/image_sorter.ex b/lib/philomena_web/image_sorter.ex index a58a9def5..f727dd723 100644 --- a/lib/philomena_web/image_sorter.ex +++ b/lib/philomena_web/image_sorter.ex @@ -14,6 +14,9 @@ defmodule PhilomenaWeb.ImageSorter do comment_count tag_count wilson_score + pixels + size + duration ) def parse_sort(params, query) do diff --git a/lib/philomena_web/templates/image/_image_meta.html.slime b/lib/philomena_web/templates/image/_image_meta.html.slime index 503f1e03c..ef7f9ef3e 100644 --- a/lib/philomena_web/templates/image/_image_meta.html.slime +++ b/lib/philomena_web/templates/image/_image_meta.html.slime @@ -61,6 +61,14 @@ = @image.image_width | x = @image.image_height + + = if not is_nil(@image.image_duration) and @image.image_is_animated and @image.image_duration > 0 do + span.image-size + |   + - dur = ceil(@image.image_duration) + - {hh, mm, ss} = {div(dur, 3600), div(rem(dur, 3600), 60), rem(dur, 60)} + = :io_lib.format("~2..0B:~2..0B:~2..0B", [hh, mm, ss]) + =<> String.upcase(to_string(@image.image_format)) - size_kb = div(@image.image_size, 1000) - size_mb = Float.round(size_kb / 1000.0, 2) diff --git a/lib/philomena_web/templates/image/new.html.slime b/lib/philomena_web/templates/image/new.html.slime index 10cae8563..2e9b92aea 100644 --- a/lib/philomena_web/templates/image/new.html.slime +++ b/lib/philomena_web/templates/image/new.html.slime @@ -36,6 +36,7 @@ = error_tag f, :image_height = error_tag f, :image_name = error_tag f, :image_mime_type + = error_tag f, :image_duration = error_tag f, :image_orig_sha512_hash .field.field--inline diff --git a/lib/philomena_web/views/api/json/image_view.ex b/lib/philomena_web/views/api/json/image_view.ex index d0985a854..ec0a9699c 100644 --- a/lib/philomena_web/views/api/json/image_view.ex +++ b/lib/philomena_web/views/api/json/image_view.ex @@ -59,6 +59,8 @@ defmodule PhilomenaWeb.Api.Json.ImageView do width: image.image_width, height: image.image_height, mime_type: image.image_mime_type, + size: image.image_size, + duration: image.image_duration, format: image.image_format, aspect_ratio: image.image_aspect_ratio, name: image.image_name, diff --git a/priv/repo/migrations/20200706171350_add_duration_to_images.exs b/priv/repo/migrations/20200706171350_add_duration_to_images.exs new file mode 100644 index 000000000..d9cd28f79 --- /dev/null +++ b/priv/repo/migrations/20200706171350_add_duration_to_images.exs @@ -0,0 +1,14 @@ +defmodule Philomena.Repo.Migrations.AddDurationToImages do + use Ecto.Migration + + def change do + alter table("images") do + add :image_duration, :float + end + + # After successful migration: + # alias Philomena.Elasticsearch + # alias Philomena.Images.Image + # Elasticsearch.update_mapping!(Image) + end +end diff --git a/priv/repo/structure.sql b/priv/repo/structure.sql index 7193f6a2d..04ce14c9f 100644 --- a/priv/repo/structure.sql +++ b/priv/repo/structure.sql @@ -882,7 +882,8 @@ CREATE TABLE public.images ( destroyed_content boolean DEFAULT false NOT NULL, hidden_image_key character varying, scratchpad character varying, - hides_count integer DEFAULT 0 NOT NULL + hides_count integer DEFAULT 0 NOT NULL, + image_duration double precision ); @@ -4706,3 +4707,4 @@ INSERT INTO public."schema_migrations" (version) VALUES (20200503002523); INSERT INTO public."schema_migrations" (version) VALUES (20200607000511); INSERT INTO public."schema_migrations" (version) VALUES (20200617111116); INSERT INTO public."schema_migrations" (version) VALUES (20200617113333); +INSERT INTO public."schema_migrations" (version) VALUES (20200706171350); From 41b3078da8851d62ab30a584457dbeb69545e98c Mon Sep 17 00:00:00 2001 From: "byte[]" Date: Mon, 6 Jul 2020 14:10:35 -0400 Subject: [PATCH 02/13] fix apng encoding --- lib/philomena/processors/png.ex | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/lib/philomena/processors/png.ex b/lib/philomena/processors/png.ex index 32d5d240e..5a536dca1 100644 --- a/lib/philomena/processors/png.ex +++ b/lib/philomena/processors/png.ex @@ -3,10 +3,11 @@ defmodule Philomena.Processors.Png do def process(analysis, file, versions) do dimensions = analysis.dimensions + animated? = analysis.animated? {:ok, intensities} = Intensities.file(file) - scaled = Enum.flat_map(versions, &scale_if_smaller(file, dimensions, &1)) + scaled = Enum.flat_map(versions, &scale_if_smaller(file, animated?, dimensions, &1)) %{ intensities: intensities, @@ -35,13 +36,13 @@ defmodule Philomena.Processors.Png do optimized end - defp scale_if_smaller(_file, _dimensions, {:full, _target_dim}) do + defp scale_if_smaller(_file, _animated?, _dimensions, {:full, _target_dim}) do [{:symlink_original, "full.png"}] end - defp scale_if_smaller(file, {width, height}, {thumb_name, {target_width, target_height}}) do + defp scale_if_smaller(file, animated?, {width, height}, {thumb_name, {target_width, target_height}}) do if width > target_width or height > target_height do - scaled = scale(file, {target_width, target_height}) + scaled = scale(file, animated?, {target_width, target_height}) [{:copy, scaled, "#{thumb_name}.png"}] else @@ -49,16 +50,22 @@ defmodule Philomena.Processors.Png do end end - defp scale(file, {width, height}) do + defp scale(file, animated?, {width, height}) do scaled = Briefly.create!(extname: ".png") scale_filter = "scale=w=#{width}:h=#{height}:force_original_aspect_ratio=decrease,format=rgb32" {_output, 0} = - System.cmd("ffmpeg", ["-loglevel", "0", "-y", "-i", file, "-vf", scale_filter, scaled]) + cond do + animated? -> + System.cmd("ffmpeg", ["-loglevel", "0", "-y", "-i", file, "-plays", "0", "-vf", scale_filter, "-f", "apng", scaled]) - {_output, 0} = System.cmd("optipng", ["-i0", "-o1", "-quiet", "-clobber", scaled]) + true -> + System.cmd("ffmpeg", ["-loglevel", "0", "-y", "-i", file, "-vf", scale_filter, scaled]) + end + + System.cmd("optipng", ["-i0", "-o1", "-quiet", "-clobber", scaled]) scaled end From c65f27a1ff1b5b75a92b41600ea97595d1a37c43 Mon Sep 17 00:00:00 2001 From: "byte[]" Date: Mon, 6 Jul 2020 14:12:18 -0400 Subject: [PATCH 03/13] mix format --- lib/philomena/elasticsearch.ex | 4 +- lib/philomena/processors/png.ex | 22 ++++++- lib/philomena/scrapers/twitter.ex | 4 +- .../image/description_controller.ex | 8 ++- .../controllers/image/source_controller.ex | 1 + .../controllers/image/tag_controller.ex | 7 +- ...0617111116_prod_schema_sync_2020_06_17.exs | 65 +++++++++++++++---- .../20200617113333_prod_schema_sync2.exs | 6 +- 8 files changed, 98 insertions(+), 19 deletions(-) diff --git a/lib/philomena/elasticsearch.ex b/lib/philomena/elasticsearch.ex index 9374ce472..62b56590a 100644 --- a/lib/philomena/elasticsearch.ex +++ b/lib/philomena/elasticsearch.ex @@ -52,7 +52,9 @@ defmodule Philomena.Elasticsearch do index_name = index.index_name() mapping = index.mapping().mappings.properties - Elastix.Mapping.put(elastic_url(), index_name, "_doc", %{properties: mapping}, include_type_name: true) + Elastix.Mapping.put(elastic_url(), index_name, "_doc", %{properties: mapping}, + include_type_name: true + ) end def index_document(doc, module) do diff --git a/lib/philomena/processors/png.ex b/lib/philomena/processors/png.ex index 5a536dca1..1b1d5fc23 100644 --- a/lib/philomena/processors/png.ex +++ b/lib/philomena/processors/png.ex @@ -40,7 +40,12 @@ defmodule Philomena.Processors.Png do [{:symlink_original, "full.png"}] end - defp scale_if_smaller(file, animated?, {width, height}, {thumb_name, {target_width, target_height}}) do + defp scale_if_smaller( + file, + animated?, + {width, height}, + {thumb_name, {target_width, target_height}} + ) do if width > target_width or height > target_height do scaled = scale(file, animated?, {target_width, target_height}) @@ -59,7 +64,20 @@ defmodule Philomena.Processors.Png do {_output, 0} = cond do animated? -> - System.cmd("ffmpeg", ["-loglevel", "0", "-y", "-i", file, "-plays", "0", "-vf", scale_filter, "-f", "apng", scaled]) + System.cmd("ffmpeg", [ + "-loglevel", + "0", + "-y", + "-i", + file, + "-plays", + "0", + "-vf", + scale_filter, + "-f", + "apng", + scaled + ]) true -> System.cmd("ffmpeg", ["-loglevel", "0", "-y", "-i", file, "-vf", scale_filter, scaled]) diff --git a/lib/philomena/scrapers/twitter.ex b/lib/philomena/scrapers/twitter.ex index 38eaeb642..36e8ea5b2 100644 --- a/lib/philomena/scrapers/twitter.ex +++ b/lib/philomena/scrapers/twitter.ex @@ -60,7 +60,9 @@ defmodule Philomena.Scrapers.Twitter do end defp extract_guest_token_and_bearer(%Tesla.Env{body: page, headers: headers}) do - [{_, gt}] = Enum.filter(headers, fn {k, v} -> k == "set-cookie" and String.starts_with?(v, "gt=") end) + [{_, gt}] = + Enum.filter(headers, fn {k, v} -> k == "set-cookie" and String.starts_with?(v, "gt=") end) + [gt] = Regex.run(@gt_regex, gt, capture: :all_but_first) [script] = Regex.run(@script_regex, page, capture: :all_but_first) diff --git a/lib/philomena_web/controllers/image/description_controller.ex b/lib/philomena_web/controllers/image/description_controller.ex index b249ba4c6..d6ea745a6 100644 --- a/lib/philomena_web/controllers/image/description_controller.ex +++ b/lib/philomena_web/controllers/image/description_controller.ex @@ -7,7 +7,12 @@ defmodule PhilomenaWeb.Image.DescriptionController do plug PhilomenaWeb.FilterBannedUsersPlug plug PhilomenaWeb.CanaryMapPlug, update: :edit_description - plug :load_and_authorize_resource, model: Image, id_name: "image_id", persisted: true, preload: [:tags, :user] + + plug :load_and_authorize_resource, + model: Image, + id_name: "image_id", + persisted: true, + preload: [:tags, :user] def update(conn, %{"image" => image_params}) do image = conn.assigns.image @@ -20,6 +25,7 @@ defmodule PhilomenaWeb.Image.DescriptionController do "image:description_update", %{image_id: image.id, added: image.description, removed: old_description} ) + PhilomenaWeb.Endpoint.broadcast!( "firehose", "image:update", diff --git a/lib/philomena_web/controllers/image/source_controller.ex b/lib/philomena_web/controllers/image/source_controller.ex index 729e74667..0d4c58cd4 100644 --- a/lib/philomena_web/controllers/image/source_controller.ex +++ b/lib/philomena_web/controllers/image/source_controller.ex @@ -26,6 +26,7 @@ defmodule PhilomenaWeb.Image.SourceController do "image:source_update", %{image_id: image.id, added: [image.source_url], removed: [old_source]} ) + PhilomenaWeb.Endpoint.broadcast!( "firehose", "image:update", diff --git a/lib/philomena_web/controllers/image/tag_controller.ex b/lib/philomena_web/controllers/image/tag_controller.ex index f80acdc7d..b987f983c 100644 --- a/lib/philomena_web/controllers/image/tag_controller.ex +++ b/lib/philomena_web/controllers/image/tag_controller.ex @@ -25,8 +25,13 @@ defmodule PhilomenaWeb.Image.TagController do PhilomenaWeb.Endpoint.broadcast!( "firehose", "image:tag_update", - %{image_id: image.id, added: Enum.map(added_tags, & &1.name), removed: Enum.map(removed_tags, & &1.name)} + %{ + image_id: image.id, + added: Enum.map(added_tags, & &1.name), + removed: Enum.map(removed_tags, & &1.name) + } ) + PhilomenaWeb.Endpoint.broadcast!( "firehose", "image:update", diff --git a/priv/repo/migrations/20200617111116_prod_schema_sync_2020_06_17.exs b/priv/repo/migrations/20200617111116_prod_schema_sync_2020_06_17.exs index 4e8d5c7ca..9fc49f66c 100644 --- a/priv/repo/migrations/20200617111116_prod_schema_sync_2020_06_17.exs +++ b/priv/repo/migrations/20200617111116_prod_schema_sync_2020_06_17.exs @@ -2,32 +2,73 @@ defmodule Philomena.Repo.Migrations.ProdSchemaSync20200617 do use Ecto.Migration def change do - execute("CREATE INDEX index_comments_on_image_id_and_created_at ON public.comments USING btree (image_id, created_at);") + execute( + "CREATE INDEX index_comments_on_image_id_and_created_at ON public.comments USING btree (image_id, created_at);" + ) + execute("DROP INDEX index_dnp_entries_on_modifying_user_id;") - execute("CREATE INDEX index_dnp_entries_on_aasm_state_filtered ON public.dnp_entries USING btree (aasm_state) WHERE ((aasm_state)::text = ANY (ARRAY[('requested'::character varying)::text, ('claimed'::character varying)::text, ('rescinded'::character varying)::text, ('acknowledged'::character varying)::text]));") - execute("CREATE INDEX index_duplicate_reports_on_state_filtered ON public.duplicate_reports USING btree (state) WHERE ((state)::text = ANY (ARRAY[('open'::character varying)::text, ('claimed'::character varying)::text]));") + + execute( + "CREATE INDEX index_dnp_entries_on_aasm_state_filtered ON public.dnp_entries USING btree (aasm_state) WHERE ((aasm_state)::text = ANY (ARRAY[('requested'::character varying)::text, ('claimed'::character varying)::text, ('rescinded'::character varying)::text, ('acknowledged'::character varying)::text]));" + ) + + execute( + "CREATE INDEX index_duplicate_reports_on_state_filtered ON public.duplicate_reports USING btree (state) WHERE ((state)::text = ANY (ARRAY[('open'::character varying)::text, ('claimed'::character varying)::text]));" + ) + execute("CREATE INDEX index_filters_on_name ON public.filters USING btree (name);") - execute("CREATE INDEX index_filters_on_system ON public.filters USING btree (system) WHERE (system = true);") - execute("CREATE UNIQUE INDEX index_gallery_interactions_on_gallery_id_and_image_id ON public.gallery_interactions USING btree (gallery_id, image_id);") - execute("CREATE INDEX index_gallery_interactions_on_gallery_id_and_position ON public.gallery_interactions USING btree (gallery_id, \"position\");") + + execute( + "CREATE INDEX index_filters_on_system ON public.filters USING btree (system) WHERE (system = true);" + ) + + execute( + "CREATE UNIQUE INDEX index_gallery_interactions_on_gallery_id_and_image_id ON public.gallery_interactions USING btree (gallery_id, image_id);" + ) + + execute( + "CREATE INDEX index_gallery_interactions_on_gallery_id_and_position ON public.gallery_interactions USING btree (gallery_id, \"position\");" + ) + execute("DROP INDEX index_images_on_first_seen_at;") execute("DROP INDEX index_notifications_on_actor_id_and_actor_type;") - execute("CREATE UNIQUE INDEX index_notifications_on_actor_id_and_actor_type ON public.notifications USING btree (actor_id, actor_type);") - execute("CREATE INDEX index_subnet_bans_on_specification ON public.subnet_bans USING gist (specification inet_ops);") - execute("CREATE INDEX index_tag_changes_on_fingerprint ON public.tag_changes USING btree (fingerprint);") - execute("CREATE INDEX index_tag_changes_on_ip ON public.tag_changes USING gist (ip inet_ops);") + + execute( + "CREATE UNIQUE INDEX index_notifications_on_actor_id_and_actor_type ON public.notifications USING btree (actor_id, actor_type);" + ) + + execute( + "CREATE INDEX index_subnet_bans_on_specification ON public.subnet_bans USING gist (specification inet_ops);" + ) + + execute( + "CREATE INDEX index_tag_changes_on_fingerprint ON public.tag_changes USING btree (fingerprint);" + ) + + execute( + "CREATE INDEX index_tag_changes_on_ip ON public.tag_changes USING gist (ip inet_ops);" + ) + execute("DROP INDEX index_tags_on_name;") execute("CREATE UNIQUE INDEX index_tags_on_name ON public.tags USING btree (name);") execute("DROP INDEX index_tags_on_slug;") execute("CREATE UNIQUE INDEX index_tags_on_slug ON public.tags USING btree (slug);") execute("DROP INDEX index_topics_on_sticky;") execute("DROP INDEX index_user_bans_on_created_at;") - execute("CREATE INDEX index_user_bans_on_created_at ON public.user_bans USING btree (created_at DESC);") + + execute( + "CREATE INDEX index_user_bans_on_created_at ON public.user_bans USING btree (created_at DESC);" + ) + execute("DROP INDEX index_users_on_name;") execute("CREATE UNIQUE INDEX index_users_on_name ON public.users USING btree (name);") execute("DROP INDEX index_users_on_slug;") execute("CREATE UNIQUE INDEX index_users_on_slug ON public.users USING btree (slug)") - execute("CREATE INDEX index_users_on_watched_tag_ids ON public.users USING gin (watched_tag_ids);") + + execute( + "CREATE INDEX index_users_on_watched_tag_ids ON public.users USING gin (watched_tag_ids);" + ) + execute("DROP INDEX index_adverts_on_live;") execute("DROP INDEX index_commissions_on_categories;") end diff --git a/priv/repo/migrations/20200617113333_prod_schema_sync2.exs b/priv/repo/migrations/20200617113333_prod_schema_sync2.exs index dc5466af5..87f787bc6 100644 --- a/priv/repo/migrations/20200617113333_prod_schema_sync2.exs +++ b/priv/repo/migrations/20200617113333_prod_schema_sync2.exs @@ -46,7 +46,11 @@ defmodule Philomena.Repo.Migrations.ProdSchemaSync2 do execute("ALTER SEQUENCE users_id_seq AS BIGINT;") execute("ALTER SEQUENCE versions_id_seq AS BIGINT;") execute("CREATE INDEX index_users_on_created_at ON public.users USING btree (created_at);") - execute("CREATE INDEX index_users_on_role ON public.users USING btree (role) WHERE ((role)::text <> 'user'::text);") + + execute( + "CREATE INDEX index_users_on_role ON public.users USING btree (role) WHERE ((role)::text <> 'user'::text);" + ) + execute("DROP INDEX temp_unique_index_tags_on_name;") execute("DROP INDEX temp_unique_index_tags_on_slug;") end From 4cdc3622370969456b2facad6d9171d0c498cf7b Mon Sep 17 00:00:00 2001 From: Christine Dodrill Date: Sat, 4 Jul 2020 13:13:05 -0400 Subject: [PATCH 04/13] philomena_web: hack in forum posts to the firehose (#2) * philomena_web: hack in forum posts to the firehose (cherry-picked from commit c5ff6f38f3c1274cd0c6160b798d6f74549735ef) --- lib/philomena_web/controllers/topic/post_controller.ex | 8 ++++++++ lib/philomena_web/controllers/topic_controller.ex | 8 ++++++++ lib/philomena_web/views/api/json/forum/topic/post_view.ex | 8 ++++++++ 3 files changed, 24 insertions(+) diff --git a/lib/philomena_web/controllers/topic/post_controller.ex b/lib/philomena_web/controllers/topic/post_controller.ex index f1234dc3e..df6b62c12 100644 --- a/lib/philomena_web/controllers/topic/post_controller.ex +++ b/lib/philomena_web/controllers/topic/post_controller.ex @@ -35,6 +35,14 @@ defmodule PhilomenaWeb.Topic.PostController do Posts.reindex_post(post) UserStatistics.inc_stat(conn.assigns.current_user, :forum_posts) + if forum.access_level == "normal" do + PhilomenaWeb.Endpoint.broadcast!( + "firehose", + "post:create", + PhilomenaWeb.Api.Json.Forum.Topic.PostView.render("firehose.json", %{post: post, topic: topic, forum: forum}) + ) + end + conn |> put_flash(:info, "Post created successfully.") |> redirect( diff --git a/lib/philomena_web/controllers/topic_controller.ex b/lib/philomena_web/controllers/topic_controller.ex index e96ab7ad1..9d7816762 100644 --- a/lib/philomena_web/controllers/topic_controller.ex +++ b/lib/philomena_web/controllers/topic_controller.ex @@ -108,6 +108,14 @@ defmodule PhilomenaWeb.TopicController do Posts.reindex_post(post) Topics.notify_topic(topic) + if forum.access_level == "normal" do + PhilomenaWeb.Endpoint.broadcast!( + "firehose", + "post:create", + PhilomenaWeb.Api.Json.Forum.Topic.PostView.render("firehose.json", %{post: post, topic: topic, forum: forum}) + ) + end + conn |> put_flash(:info, "Successfully posted topic.") |> redirect(to: Routes.forum_topic_path(conn, :show, forum, topic)) diff --git a/lib/philomena_web/views/api/json/forum/topic/post_view.ex b/lib/philomena_web/views/api/json/forum/topic/post_view.ex index ef69e3d66..4d6ecdcb0 100644 --- a/lib/philomena_web/views/api/json/forum/topic/post_view.ex +++ b/lib/philomena_web/views/api/json/forum/topic/post_view.ex @@ -13,6 +13,14 @@ defmodule PhilomenaWeb.Api.Json.Forum.Topic.PostView do %{post: render_one(post, PhilomenaWeb.Api.Json.Forum.Topic.PostView, "post.json", assigns)} end + def render("firehose.json", %{post: post, topic: topic, forum: forum} = assigns) do + %{ + post: render_one(post, PhilomenaWeb.Api.Json.Forum.Topic.PostView, "post.json", assigns), + topic: render_one(topic, PhilomenaWeb.Api.Json.Forum.TopicView, "topic.json", assigns), + forum: render_one(forum, PhilomenaWeb.Api.Json.ForumView, "forum.json", assigns) + } + end + def render("post.json", %{post: %{topic: %{hidden_from_users: true}} = post}) do %{ id: post.id, From 70e70254edfdc5e5823d00809ebb4546747ff485 Mon Sep 17 00:00:00 2001 From: "byte[]" Date: Mon, 6 Jul 2020 14:20:50 -0400 Subject: [PATCH 05/13] mix format --- lib/philomena_web/controllers/topic/post_controller.ex | 6 +++++- lib/philomena_web/controllers/topic_controller.ex | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/philomena_web/controllers/topic/post_controller.ex b/lib/philomena_web/controllers/topic/post_controller.ex index df6b62c12..535c6cff6 100644 --- a/lib/philomena_web/controllers/topic/post_controller.ex +++ b/lib/philomena_web/controllers/topic/post_controller.ex @@ -39,7 +39,11 @@ defmodule PhilomenaWeb.Topic.PostController do PhilomenaWeb.Endpoint.broadcast!( "firehose", "post:create", - PhilomenaWeb.Api.Json.Forum.Topic.PostView.render("firehose.json", %{post: post, topic: topic, forum: forum}) + PhilomenaWeb.Api.Json.Forum.Topic.PostView.render("firehose.json", %{ + post: post, + topic: topic, + forum: forum + }) ) end diff --git a/lib/philomena_web/controllers/topic_controller.ex b/lib/philomena_web/controllers/topic_controller.ex index 9d7816762..79d16d64c 100644 --- a/lib/philomena_web/controllers/topic_controller.ex +++ b/lib/philomena_web/controllers/topic_controller.ex @@ -112,7 +112,11 @@ defmodule PhilomenaWeb.TopicController do PhilomenaWeb.Endpoint.broadcast!( "firehose", "post:create", - PhilomenaWeb.Api.Json.Forum.Topic.PostView.render("firehose.json", %{post: post, topic: topic, forum: forum}) + PhilomenaWeb.Api.Json.Forum.Topic.PostView.render("firehose.json", %{ + post: post, + topic: topic, + forum: forum + }) ) end From e581bc2d4ef12b3490126179df807589fcab513a Mon Sep 17 00:00:00 2001 From: "byte[]" Date: Mon, 6 Jul 2020 14:42:10 -0400 Subject: [PATCH 06/13] ensure animation info is present in api --- lib/philomena_web/views/api/json/image_view.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/philomena_web/views/api/json/image_view.ex b/lib/philomena_web/views/api/json/image_view.ex index ec0a9699c..abfc347ee 100644 --- a/lib/philomena_web/views/api/json/image_view.ex +++ b/lib/philomena_web/views/api/json/image_view.ex @@ -61,6 +61,7 @@ defmodule PhilomenaWeb.Api.Json.ImageView do mime_type: image.image_mime_type, size: image.image_size, duration: image.image_duration, + animated: image.image_is_animated, format: image.image_format, aspect_ratio: image.image_aspect_ratio, name: image.image_name, From 4e70124f3681e7135fc79318e6c19a59c4045caf Mon Sep 17 00:00:00 2001 From: "byte[]" Date: Wed, 8 Jul 2020 00:26:58 -0400 Subject: [PATCH 07/13] fix twitter scraper --- lib/philomena/scrapers/twitter.ex | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/lib/philomena/scrapers/twitter.ex b/lib/philomena/scrapers/twitter.ex index 36e8ea5b2..631cda2ee 100644 --- a/lib/philomena/scrapers/twitter.ex +++ b/lib/philomena/scrapers/twitter.ex @@ -1,5 +1,5 @@ defmodule Philomena.Scrapers.Twitter do - @gt_regex ~r|gt=(\d+?);| + @gt_regex ~r|document.cookie = decodeURIComponent\("gt=(\d+);| @url_regex ~r|\Ahttps?://(?:mobile\.)?twitter.com/([A-Za-z\d_]+)/status/([\d]+)/?| @script_regex ~r|