Rework scaled dimensions and compression (#737)

* Rework scaled dimensions and compression

* Unbreak image / banner uploads

---------

Co-authored-by: Paul Frazee <pfrazee@gmail.com>

src/lib/api/index.ts

@@ -110,6 +110,7 @@ export async function post(store: RootStoreModel, opts: PostOpts) {
     const images: AppBskyEmbedImages.Image[] = []
     for (const image of opts.images) {
       opts.onStateChange?.(`Uploading image #${images.length + 1}...`)
+      await image.compress()
       const path = image.compressed?.path ?? image.path
       const res = await uploadBlob(store, path, 'image/jpeg')
       images.push({

src/lib/media/manip.ts

@@ -6,52 +6,8 @@ import * as RNFS from 'react-native-fs'
 import uuid from 'react-native-uuid'
 import * as Sharing from 'expo-sharing'
 import {Dimensions} from './types'
-import {POST_IMG_MAX} from 'lib/constants'
 import {isAndroid, isIOS} from 'platform/detection'
 
-export async function compressAndResizeImageForPost(
-  image: Image,
-): Promise<Image> {
-  const uri = `file://${image.path}`
-  let resized: Omit<Image, 'mime'>
-
-  for (let i = 0; i < 9; i++) {
-    const quality = 100 - i * 10
-
-    try {
-      resized = await ImageResizer.createResizedImage(
-        uri,
-        POST_IMG_MAX.width,
-        POST_IMG_MAX.height,
-        'JPEG',
-        quality,
-        undefined,
-        undefined,
-        undefined,
-        {mode: 'cover'},
-      )
-    } catch (err) {
-      throw new Error(`Failed to resize: ${err}`)
-    }
-
-    if (resized.size < POST_IMG_MAX.size) {
-      const path = await moveToPermanentPath(resized.path)
-
-      return {
-        path,
-        mime: 'image/jpeg',
-        size: resized.size,
-        height: resized.height,
-        width: resized.width,
-      }
-    }
-  }
-
-  throw new Error(
-    `This image is too big! We couldn't compress it down to ${POST_IMG_MAX.size} bytes`,
-  )
-}
-
 export async function compressIfNeeded(
   img: Image,
   maxSize: number = 1000000,

src/lib/media/manip.web.ts

@@ -1,25 +1,6 @@
 import {Dimensions} from './types'
 import {Image as RNImage} from 'react-native-image-crop-picker'
 import {getDataUriSize, blobToDataUri} from './util'
-import {POST_IMG_MAX} from 'lib/constants'
-
-export async function compressAndResizeImageForPost({
-  path,
-  width,
-  height,
-}: {
-  path: string
-  width: number
-  height: number
-}): Promise<RNImage> {
-  // Compression is handled in `doResize` via `quality`
-  return await doResize(path, {
-    width,
-    height,
-    maxSize: POST_IMG_MAX.size,
-    mode: 'stretch',
-  })
-}
 
 export async function compressIfNeeded(
   img: RNImage,

src/lib/media/picker.e2e.tsx

@@ -2,7 +2,7 @@ import {RootStoreModel} from 'state/index'
 import {Image as RNImage} from 'react-native-image-crop-picker'
 import RNFS from 'react-native-fs'
 import {CropperOptions} from './types'
-import {compressAndResizeImageForPost} from './manip'
+import {compressIfNeeded} from './manip'
 
 let _imageCounter = 0
 async function getFile() {
@@ -13,7 +13,7 @@ async function getFile() {
       .join('/'),
   )
   const file = files[_imageCounter++ % files.length]
-  return await compressAndResizeImageForPost({
+  return await compressIfNeeded({
     path: file.path,
     mime: 'image/jpeg',
     size: file.size,

src/lib/media/util.ts

@@ -1,5 +1,3 @@
-import {Dimensions} from './types'
-
 export function extractDataUriMime(uri: string): string {
   return uri.substring(uri.indexOf(':') + 1, uri.indexOf(';'))
 }
@@ -10,21 +8,6 @@ export function getDataUriSize(uri: string): number {
   return Math.round((uri.length * 3) / 4)
 }
 
-export function scaleDownDimensions(
-  dim: Dimensions,
-  max: Dimensions,
-): Dimensions {
-  if (dim.width < max.width && dim.height < max.height) {
-    return dim
-  }
-  const wScale = dim.width > max.width ? max.width / dim.width : 1
-  const hScale = dim.height > max.height ? max.height / dim.height : 1
-  if (wScale < hScale) {
-    return {width: dim.width * wScale, height: dim.height * wScale}
-  }
-  return {width: dim.width * hScale, height: dim.height * hScale}
-}
-
 export function isUriImage(uri: string) {
   return /\.(jpg|jpeg|png).*$/.test(uri)
 }

src/state/models/cache/image-sizes.ts

@@ -16,6 +16,7 @@ export class ImageSizesCache {
     if (Dimensions) {
       return Dimensions
     }
+
     const prom =
       this.activeRequests.get(uri) ||
       new Promise<Dimensions>(resolve => {

src/state/models/media/gallery.ts

@@ -4,7 +4,6 @@ import {ImageModel} from './image'
 import {Image as RNImage} from 'react-native-image-crop-picker'
 import {openPicker} from 'lib/media/picker'
 import {getImageDim} from 'lib/media/manip'
-import {getDataUriSize} from 'lib/media/util'
 import {isNative} from 'platform/detection'
 
 export class GalleryModel {
@@ -24,13 +23,7 @@ export class GalleryModel {
     return this.images.length
   }
 
-  get paths() {
+  async add(image_: Omit<RNImage, 'size'>) {
-    return this.images.map(image =>
-      image.compressed === undefined ? image.path : image.compressed.path,
-    )
-  }
-
-  async add(image_: RNImage) {
     if (this.size >= 4) {
       return
     }
@@ -39,15 +32,9 @@ export class GalleryModel {
     if (!this.images.some(i => i.path === image_.path)) {
       const image = new ImageModel(this.rootStore, image_)
 
-      if (!isNative) {
+      // Initial resize
-        await image.manipulate({})
+      image.manipulate({})
-      } else {
+      this.images.push(image)
-        await image.compress()
-      }
-
-      runInAction(() => {
-        this.images.push(image)
-      })
     }
   }
 
@@ -70,11 +57,10 @@ export class GalleryModel {
 
     const {width, height} = await getImageDim(uri)
 
-    const image: RNImage = {
+    const image = {
       path: uri,
       height,
       width,
-      size: getDataUriSize(uri),
       mime: 'image/jpeg',
     }
 

src/state/models/media/image.ts

@@ -3,14 +3,11 @@ import {RootStoreModel} from 'state/index'
 import {makeAutoObservable, runInAction} from 'mobx'
 import {POST_IMG_MAX} from 'lib/constants'
 import * as ImageManipulator from 'expo-image-manipulator'
-import {getDataUriSize, scaleDownDimensions} from 'lib/media/util'
+import {getDataUriSize} from 'lib/media/util'
 import {openCropper} from 'lib/media/picker'
 import {ActionCrop, FlipType, SaveFormat} from 'expo-image-manipulator'
 import {Position} from 'react-avatar-editor'
-import {compressAndResizeImageForPost} from 'lib/media/manip'
+import {Dimensions} from 'lib/media/types'
-
-// TODO: EXIF embed
-// Cases to consider: ExternalEmbed
 
 export interface ImageManipulationAttributes {
   aspectRatio?: '4:3' | '1:1' | '3:4' | 'None'
@@ -21,17 +18,16 @@ export interface ImageManipulationAttributes {
   flipVertical?: boolean
 }
 
-export class ImageModel implements RNImage {
+const MAX_IMAGE_SIZE_IN_BYTES = 976560
+
+export class ImageModel implements Omit<RNImage, 'size'> {
   path: string
   mime = 'image/jpeg'
   width: number
   height: number
-  size: number
   altText = ''
   cropped?: RNImage = undefined
   compressed?: RNImage = undefined
-  scaledWidth: number = POST_IMG_MAX.width
-  scaledHeight: number = POST_IMG_MAX.height
 
   // Web manipulation
   prev?: RNImage
@@ -44,7 +40,7 @@ export class ImageModel implements RNImage {
   }
   prevAttributes: ImageManipulationAttributes = {}
 
-  constructor(public rootStore: RootStoreModel, image: RNImage) {
+  constructor(public rootStore: RootStoreModel, image: Omit<RNImage, 'size'>) {
     makeAutoObservable(this, {
       rootStore: false,
     })
@@ -52,19 +48,8 @@ export class ImageModel implements RNImage {
     this.path = image.path
     this.width = image.width
     this.height = image.height
-    this.size = image.size
-    this.calcScaledDimensions()
   }
 
-  // TODO: Revisit compression factor due to updated sizing with zoom
-  // get compressionFactor() {
-  //   const MAX_IMAGE_SIZE_IN_BYTES = 976560
-
-  //   return this.size < MAX_IMAGE_SIZE_IN_BYTES
-  //     ? 1
-  //     : MAX_IMAGE_SIZE_IN_BYTES / this.size
-  // }
-
   setRatio(aspectRatio: ImageManipulationAttributes['aspectRatio']) {
     this.attributes.aspectRatio = aspectRatio
   }
@@ -93,8 +78,24 @@ export class ImageModel implements RNImage {
     }
   }
 
-  getDisplayDimensions(
+  getUploadDimensions(
-    as: ImageManipulationAttributes['aspectRatio'] = '1:1',
+    dimensions: Dimensions,
+    maxDimensions: Dimensions = POST_IMG_MAX,
+    as: ImageManipulationAttributes['aspectRatio'] = 'None',
+  ) {
+    const {width, height} = dimensions
+    const {width: maxWidth, height: maxHeight} = maxDimensions
+
+    return width < maxWidth && height < maxHeight
+      ? {
+          width,
+          height,
+        }
+      : this.getResizedDimensions(as, POST_IMG_MAX.width)
+  }
+
+  getResizedDimensions(
+    as: ImageManipulationAttributes['aspectRatio'] = 'None',
     maxSide: number,
   ) {
     const ratioMultiplier = this.ratioMultipliers[as]
@@ -119,59 +120,70 @@ export class ImageModel implements RNImage {
     }
   }
 
-  calcScaledDimensions() {
-    const {width, height} = scaleDownDimensions(
-      {width: this.width, height: this.height},
-      POST_IMG_MAX,
-    )
-    this.scaledWidth = width
-    this.scaledHeight = height
-  }
-
   async setAltText(altText: string) {
     this.altText = altText
   }
 
-  // Only for mobile
+  // Only compress prior to upload
+  async compress() {
+    for (let i = 10; i > 0; i--) {
+      // Float precision
+      const factor = Math.round(i) / 10
+      const compressed = await ImageManipulator.manipulateAsync(
+        this.cropped?.path ?? this.path,
+        undefined,
+        {
+          compress: factor,
+          base64: true,
+          format: SaveFormat.JPEG,
+        },
+      )
+
+      if (compressed.base64 !== undefined) {
+        const size = getDataUriSize(compressed.base64)
+
+        if (size < MAX_IMAGE_SIZE_IN_BYTES) {
+          runInAction(() => {
+            this.compressed = {
+              mime: 'image/jpeg',
+              path: compressed.uri,
+              size,
+              ...compressed,
+            }
+          })
+          return
+        }
+      }
+    }
+
+    // Compression fails when removing redundant information is not possible.
+    // This can be tested with images that have high variance in noise.
+    throw new Error('Failed to compress image')
+  }
+
+  // Mobile
   async crop() {
     try {
+      // openCropper requires an output width and height hence
+      // getting upload dimensions before cropping is necessary.
+      const {width, height} = this.getUploadDimensions({
+        width: this.width,
+        height: this.height,
+      })
+
       const cropped = await openCropper(this.rootStore, {
         mediaType: 'photo',
         path: this.path,
         freeStyleCropEnabled: true,
-        width: this.scaledWidth,
-        height: this.scaledHeight,
-      })
-      runInAction(() => {
-        this.cropped = cropped
-        this.compress()
-      })
-    } catch (err) {
-      this.rootStore.log.error('Failed to crop photo', err)
-    }
-  }
-
-  async compress() {
-    try {
-      const {width, height} = scaleDownDimensions(
-        this.cropped
-          ? {width: this.cropped.width, height: this.cropped.height}
-          : {width: this.width, height: this.height},
-        POST_IMG_MAX,
-      )
-
-      // TODO: Revisit this - currently iOS uses this as well
-      const compressed = await compressAndResizeImageForPost({
-        ...(this.cropped === undefined ? this : this.cropped),
         width,
         height,
       })
 
       runInAction(() => {
-        this.compressed = compressed
+        this.cropped = cropped
       })
     } catch (err) {
-      this.rootStore.log.error('Failed to compress photo', err)
+      this.rootStore.log.error('Failed to crop photo', err)
     }
   }
 
@@ -181,6 +193,9 @@ export class ImageModel implements RNImage {
       crop?: ActionCrop['crop']
     } & ImageManipulationAttributes,
   ) {
+    let uploadWidth: number | undefined
+    let uploadHeight: number | undefined
+
     const {aspectRatio, crop, position, scale} = attributes
     const modifiers = []
 
@@ -197,14 +212,34 @@ export class ImageModel implements RNImage {
     }
 
     if (crop !== undefined) {
+      const croppedHeight = crop.height * this.height
+      const croppedWidth = crop.width * this.width
       modifiers.push({
         crop: {
           originX: crop.originX * this.width,
           originY: crop.originY * this.height,
-          height: crop.height * this.height,
+          height: croppedHeight,
-          width: crop.width * this.width,
+          width: croppedWidth,
         },
       })
+
+      const uploadDimensions = this.getUploadDimensions(
+        {width: croppedWidth, height: croppedHeight},
+        POST_IMG_MAX,
+        aspectRatio,
+      )
+
+      uploadWidth = uploadDimensions.width
+      uploadHeight = uploadDimensions.height
+    } else {
+      const uploadDimensions = this.getUploadDimensions(
+        {width: this.width, height: this.height},
+        POST_IMG_MAX,
+        aspectRatio,
+      )
+
+      uploadWidth = uploadDimensions.width
+      uploadHeight = uploadDimensions.height
     }
 
     if (scale !== undefined) {
@@ -222,36 +257,40 @@ export class ImageModel implements RNImage {
     const ratioMultiplier =
       this.ratioMultipliers[this.attributes.aspectRatio ?? '1:1']
 
-    const MAX_SIDE = 2000
-
     const result = await ImageManipulator.manipulateAsync(
       this.path,
       [
         ...modifiers,
-        {resize: ratioMultiplier > 1 ? {width: MAX_SIDE} : {height: MAX_SIDE}},
+        {
+          resize:
+            ratioMultiplier > 1 ? {width: uploadWidth} : {height: uploadHeight},
+        },
       ],
       {
-        compress: 0.9,
+        base64: true,
         format: SaveFormat.JPEG,
       },
     )
 
     runInAction(() => {
-      this.compressed = {
+      this.cropped = {
         mime: 'image/jpeg',
         path: result.uri,
-        size: getDataUriSize(result.uri),
+        size:
+          result.base64 !== undefined
+            ? getDataUriSize(result.base64)
+            : MAX_IMAGE_SIZE_IN_BYTES + 999, // shouldn't hit this unless manipulation fails
         ...result,
       }
     })
   }
 
-  resetCompressed() {
+  resetCropped() {
     this.manipulate({})
   }
 
   previous() {
-    this.compressed = this.prev
+    this.cropped = this.prev
     this.attributes = this.prevAttributes
   }
 }

src/view/com/composer/photos/Gallery.tsx

@@ -104,63 +104,61 @@ export const Gallery = observer(function ({gallery}: Props) {
 
   return !gallery.isEmpty ? (
     <View testID="selectedPhotosView" style={styles.gallery}>
-      {gallery.images.map(image =>
+      {gallery.images.map(image => (
-        image.compressed !== undefined ? (
+        <View key={`selected-image-${image.path}`} style={[imageStyle]}>
-          <View key={`selected-image-${image.path}`} style={[imageStyle]}>
+          <TouchableOpacity
+            testID="altTextButton"
+            accessibilityRole="button"
+            accessibilityLabel="Add alt text"
+            accessibilityHint=""
+            onPress={() => {
+              handleAddImageAltText(image)
+            }}
+            style={imageControlLabelStyle}>
+            <Text style={styles.imageControlTextContent}>ALT</Text>
+          </TouchableOpacity>
+          <View style={imageControlsSubgroupStyle}>
             <TouchableOpacity
-              testID="altTextButton"
+              testID="editPhotoButton"
               accessibilityRole="button"
-              accessibilityLabel="Add alt text"
+              accessibilityLabel="Edit image"
               accessibilityHint=""
               onPress={() => {
-                handleAddImageAltText(image)
+                handleEditPhoto(image)
               }}
-              style={imageControlLabelStyle}>
+              style={styles.imageControl}>
-              <Text style={styles.imageControlTextContent}>ALT</Text>
+              <FontAwesomeIcon
+                icon="pen"
+                size={12}
+                style={{color: colors.white}}
+              />
+            </TouchableOpacity>
+            <TouchableOpacity
+              testID="removePhotoButton"
+              accessibilityRole="button"
+              accessibilityLabel="Remove image"
+              accessibilityHint=""
+              onPress={() => handleRemovePhoto(image)}
+              style={styles.imageControl}>
+              <FontAwesomeIcon
+                icon="xmark"
+                size={16}
+                style={{color: colors.white}}
+              />
             </TouchableOpacity>
-            <View style={imageControlsSubgroupStyle}>
-              <TouchableOpacity
-                testID="editPhotoButton"
-                accessibilityRole="button"
-                accessibilityLabel="Edit image"
-                accessibilityHint=""
-                onPress={() => {
-                  handleEditPhoto(image)
-                }}
-                style={styles.imageControl}>
-                <FontAwesomeIcon
-                  icon="pen"
-                  size={12}
-                  style={{color: colors.white}}
-                />
-              </TouchableOpacity>
-              <TouchableOpacity
-                testID="removePhotoButton"
-                accessibilityRole="button"
-                accessibilityLabel="Remove image"
-                accessibilityHint=""
-                onPress={() => handleRemovePhoto(image)}
-                style={styles.imageControl}>
-                <FontAwesomeIcon
-                  icon="xmark"
-                  size={16}
-                  style={{color: colors.white}}
-                />
-              </TouchableOpacity>
-            </View>
-
-            <Image
-              testID="selectedPhotoImage"
-              style={[styles.image, imageStyle] as ImageStyle}
-              source={{
-                uri: image.compressed.path,
-              }}
-              accessible={true}
-              accessibilityIgnoresInvertColors
-            />
           </View>
-        ) : null,
+
-      )}
+          <Image
+            testID="selectedPhotoImage"
+            style={[styles.image, imageStyle] as ImageStyle}
+            source={{
+              uri: image.cropped?.path ?? image.path,
+            }}
+            accessible={true}
+            accessibilityIgnoresInvertColors
+          />
+        </View>
+      ))}
     </View>
   ) : null
 })

src/view/com/modals/EditImage.tsx

@@ -118,9 +118,9 @@ export const Component = observer(function ({image, gallery}: Props) {
   )
 
   useEffect(() => {
-    image.prev = image.compressed
+    image.prev = image.cropped
     image.prevAttributes = image.attributes
-    image.resetCompressed()
+    image.resetCropped()
   }, [image])
 
   const onCloseModal = useCallback(() => {
@@ -152,7 +152,7 @@ export const Component = observer(function ({image, gallery}: Props) {
         : {}),
     })
 
-    image.prev = image.compressed
+    image.prev = image.cropped
     image.prevAttributes = image.attributes
     onCloseModal()
   }, [altText, image, position, scale, onCloseModal])
@@ -168,8 +168,7 @@ export const Component = observer(function ({image, gallery}: Props) {
     }
   }, [])
 
-  // Prevents preliminary flash when transformations are being applied
+  if (image.cropped === undefined) {
-  if (image.compressed === undefined) {
     return null
   }
 
@@ -177,7 +176,7 @@ export const Component = observer(function ({image, gallery}: Props) {
     windowDimensions.width > 500 ? 410 : windowDimensions.width - 80
   const sideLength = isDesktopWeb ? 300 : computedWidth
 
-  const dimensions = image.getDisplayDimensions(aspectRatio, sideLength)
+  const dimensions = image.getResizedDimensions(aspectRatio, sideLength)
   const imgContainerStyles = {width: sideLength, height: sideLength}
 
   const imgControlStyles = {
@@ -196,7 +195,7 @@ export const Component = observer(function ({image, gallery}: Props) {
             <ImageEditor
               ref={editorRef}
               style={styles.imgEditor}
-              image={image.compressed.path}
+              image={image.cropped.path}
               scale={scale}
               border={0}
               position={position}

src/view/com/modals/Modal.tsx

@@ -15,6 +15,7 @@ import * as RepostModal from './Repost'
 import * as CreateOrEditMuteListModal from './CreateOrEditMuteList'
 import * as ListAddRemoveUserModal from './ListAddRemoveUser'
 import * as AltImageModal from './AltImage'
+import * as EditImageModal from './AltImage'
 import * as ReportAccountModal from './ReportAccount'
 import * as DeleteAccountModal from './DeleteAccount'
 import * as ChangeHandleModal from './ChangeHandle'
@@ -83,6 +84,9 @@ export const ModalsContainer = observer(function ModalsContainer() {
   } else if (activeModal?.name === 'alt-text-image') {
     snapPoints = AltImageModal.snapPoints
     element = <AltImageModal.Component {...activeModal} />
+  } else if (activeModal?.name === 'edit-image') {
+    snapPoints = AltImageModal.snapPoints
+    element = <EditImageModal.Component {...activeModal} />
   } else if (activeModal?.name === 'change-handle') {
     snapPoints = ChangeHandleModal.snapPoints
     element = <ChangeHandleModal.Component {...activeModal} />

src/view/com/modals/crop-image/CropImage.tsx

@@ -1,11 +0,0 @@
-/**
- * NOTE
- * This modal is used only in the web build
- * Native uses a third-party library
- */
-
-export const snapPoints = ['0%']
-
-export function Component() {
-  return null
-}