[FEAT/#46] ocr / Google ML Kit OCR 기능 구현

app/build.gradle

@@ -5,6 +5,7 @@ plugins {
     id 'dagger.hilt.android.plugin'
     id 'org.jetbrains.kotlin.plugin.serialization' version '1.7.10'
     id "org.jlleitschuh.gradle.ktlint" version "10.3.0"
+    id 'com.google.gms.google-services'
 }
 
 apply plugin: 'kotlin-kapt'
@@ -107,6 +108,7 @@ dependencies {
 
     // google ml kit
     implementation 'com.google.android.gms:play-services-mlkit-text-recognition:18.0.2'
+    implementation 'com.google.mlkit:text-recognition-korean:16.0.0-beta6'
 
     // viewpager2
     implementation 'androidx.viewpager2:viewpager2:1.1.0-beta01'
@@ -115,10 +117,14 @@ dependencies {
     implementation "com.tbuonomo:dotsindicator:4.3"
 
     // shared preference
-    implementation 'androidx.security:security-crypto-ktx:1.1.0-alpha03'
+    implementation 'androidx.security:security-crypto-ktx:1.1.0-alpha04'
+
+    // firebase storage
+    implementation 'com.google.firebase:firebase-core:21.1.1'
+    implementation 'com.google.firebase:firebase-storage:20.1.0'
 
     implementation 'androidx.core:core-ktx:1.9.0'
-    implementation 'androidx.appcompat:appcompat:1.5.1'
+    implementation 'androidx.appcompat:appcompat:1.6.0'
     implementation 'com.google.android.material:material:1.7.0'
     implementation 'androidx.constraintlayout:constraintlayout:2.1.4'
     testImplementation 'junit:junit:4.13.2'

app/src/main/java/com/keyneez/data/model/response/ResponseIdDto.kt

@@ -22,6 +22,6 @@ data class ResponseIdDto(
     @Serializable
     data class Character(
         val character: String?,
-        @SerialName("character_img") val characterImg: String?
+//        @SerialName("character_img") val characterImg: String?
     )
 }

app/src/main/java/com/keyneez/presentation/login/phone/PhoneFragment.kt

@@ -34,6 +34,7 @@ class PhoneFragment : BindingFragment<FragmentPhoneBinding>(R.layout.fragment_ph
 
     private fun initNextBtnClickListener() {
         binding.btnPhoneNext.setOnSingleClickListener {
+            requireContext().hideKeyboard(requireView())
             (activity as LoginActivity).setPhoneNumber(viewModel.phoneNumber.value.toString())
             (activity as LoginActivity).intentToNextPage()
         }

app/src/main/java/com/keyneez/presentation/login/pin/LoginPinFragment.kt

@@ -64,16 +64,22 @@ class LoginPinFragment : BindingFragment<FragmentLoginPinBinding>(R.layout.fragm
                 }
                 is UiState.Failure -> {
                     when (it.code) {
-                        UNAUTHORIZED_USER_CODE -> requireContext().showSnackbar(
-                            binding.root,
-                            getString(
-                                R.string.login_pin_unauthorized_msg
+                        UNAUTHORIZED_USER_CODE -> {
+                            viewModel.resetPassword()
+                            requireContext().showSnackbar(
+                                binding.root,
+                                getString(
+                                    R.string.signup_pin_confirm_invalid_pwd_msg
+                                )
                             )
-                        )
-                        INVALID_PWD_CODE -> requireContext().showSnackbar(
-                            binding.root,
-                            getString(R.string.signup_pin_confirm_invalid_pwd_msg)
-                        )
+                        }
+                        INVALID_PWD_CODE -> {
+                            viewModel.resetPassword()
+                            requireContext().showSnackbar(
+                                binding.root,
+                                getString(R.string.signup_pin_confirm_invalid_pwd_msg)
+                            )
+                        }
                         else -> requireContext().showSnackbar(
                             binding.root,
                             getString(R.string.msg_error)

app/src/main/java/com/keyneez/presentation/main/id/IdFragment.kt

@@ -29,6 +29,8 @@ class IdFragment : BindingFragment<FragmentIdBinding>(R.layout.fragment_id) {
         binding.vm = viewModel
         initBottomSheet()
         initIdPhotoBtnClickListener()
+        initIdIssueBtnClickListener()
+        initIdMainBtnClickListener()
         observeIdStateMessage()
     }
 
@@ -39,12 +41,10 @@ class IdFragment : BindingFragment<FragmentIdBinding>(R.layout.fragment_id) {
                     // 발급하기 화면이 뜨게
                     binding.layoutIdIssue.visibility = View.VISIBLE
                     binding.layoutIdMain.visibility = View.GONE
-                    initIdIssueBtnClickListener()
                 } else {
                     // 메인 아이디 화면이 뜨게
                     binding.layoutIdIssue.visibility = View.GONE
                     binding.layoutIdMain.visibility = View.VISIBLE
-                    initIdMainBtnClickListener()
                     initIdBackGround()
                 }
                 is UiState.Failure -> requireContext().showSnackbar(

app/src/main/java/com/keyneez/presentation/ocr/OcrActivity.kt

@@ -1,26 +1,35 @@
 package com.keyneez.presentation.ocr
 
+import android.graphics.Bitmap
+import android.graphics.BitmapFactory
 import android.graphics.Paint
 import android.os.Bundle
 import android.view.View
 import androidx.activity.viewModels
-import androidx.camera.core.CameraSelector
-import androidx.camera.core.Preview
+import androidx.camera.core.* // ktlint-disable no-wildcard-imports
 import androidx.camera.lifecycle.ProcessCameraProvider
 import androidx.core.content.ContextCompat
+import com.google.mlkit.vision.common.InputImage
+import com.google.mlkit.vision.text.Text
+import com.google.mlkit.vision.text.TextRecognition
+import com.google.mlkit.vision.text.korean.KoreanTextRecognizerOptions
 import com.keyneez.presentation.ocr.dialog.OcrResultFragment
 import com.keyneez.util.binding.BindingActivity
 import com.keyneez.util.extension.setOnSingleClickListener
+import com.keyneez.util.extension.showSnackbar
 import com.lab.keyneez.R
 import com.lab.keyneez.databinding.ActivityOcrBinding
 import timber.log.Timber
+import java.nio.ByteBuffer
 import java.util.concurrent.ExecutorService
 import java.util.concurrent.Executors
 
 class OcrActivity : BindingActivity<ActivityOcrBinding>(R.layout.activity_ocr) {
-    private val viewModel by viewModels<OcrViewModel>()
+    val viewModel by viewModels<OcrViewModel>()
 
     private lateinit var cameraExecutor: ExecutorService
+    private lateinit var cameraProvider: ProcessCameraProvider
+    private var imageCapture: ImageCapture? = null
 
     override fun onCreate(savedInstanceState: Bundle?) {
         super.onCreate(savedInstanceState)
@@ -41,18 +50,21 @@ class OcrActivity : BindingActivity<ActivityOcrBinding>(R.layout.activity_ocr) {
         cameraProviderFuture.addListener(
             {
                 // bind camera lifecycle
-                val cameraProvider: ProcessCameraProvider = cameraProviderFuture.get()
+                cameraProvider = cameraProviderFuture.get()
 
                 val preview = Preview.Builder().build().also {
                     it.setSurfaceProvider(binding.previewOcr.surfaceProvider)
                 }
 
+                imageCapture = ImageCapture.Builder()
+                    .build()
+
                 // select default back camera
                 val cameraSelector = CameraSelector.DEFAULT_BACK_CAMERA
 
                 try {
                     cameraProvider.unbindAll()
-                    cameraProvider.bindToLifecycle(this, cameraSelector, preview)
+                    cameraProvider.bindToLifecycle(this, cameraSelector, preview, imageCapture)
                 } catch (e: Exception) {
                     Timber.e("$e : Use case binding failed")
                 }
@@ -77,6 +89,90 @@ class OcrActivity : BindingActivity<ActivityOcrBinding>(R.layout.activity_ocr) {
 
     private fun initCameraBtnClickListener() {
         binding.btnOcrCamera.setOnSingleClickListener {
+            takePhoto()
+        }
+    }
+
+    private fun takePhoto() {
+        imageCapture = imageCapture ?: return
+
+        imageCapture!!.takePicture(
+            cameraExecutor,
+            object : ImageCapture.OnImageCapturedCallback() {
+                override fun onCaptureSuccess(image: ImageProxy) {
+                    val bitmap = imageProxyToBitmap(image)
+                    runTextRecognition(bitmap)
+                    super.onCaptureSuccess(image)
+                }
+
+                override fun onError(exception: ImageCaptureException) {
+                    Timber.tag(tag).e("exception : $exception")
+                    showSnackbar(binding.root, getString(R.string.msg_error))
+                    super.onError(exception)
+                }
+            }
+        )
+    }
+
+    private fun imageProxyToBitmap(image: ImageProxy): Bitmap {
+        val planeProxy = image.planes[0]
+        val buffer: ByteBuffer = planeProxy.buffer
+        val bytes = ByteArray(buffer.remaining())
+        buffer.get(bytes)
+        return BitmapFactory.decodeByteArray(bytes, 0, bytes.size)
+    }
+
+    private fun runTextRecognition(img: Bitmap) {
+        // 이미지 유형 : Bitmap, media.Image, ByteBuffer, byte array, device file
+        val image = InputImage.fromBitmap(img, 0)
+        val recognizer = TextRecognition.getClient(KoreanTextRecognizerOptions.Builder().build())
+        recognizer.process(image)
+            .addOnSuccessListener { visionText ->
+                processTextRecognitionResult(visionText)
+            }
+            .addOnFailureListener { e ->
+                Timber.tag(tag).e("error : $e")
+                showSnackbar(binding.root, getString(R.string.msg_error))
+            }
+    }
+
+    private fun processTextRecognitionResult(text: Text) {
+        if (text.textBlocks.size == 0) {
+            Timber.tag(tag).e("인식된 글자 없음")
+            showSnackbar(binding.root, "인식된 글자가 없습니다.")
+            return
+        }
+
+        viewModel.resetIdInfo()
+        var isSuccess = false
+
+        for (block in text.textBlocks) {
+            for (line in block.lines) {
+                for (element in line.elements) {
+                    val word = element.text
+                    Timber.tag(tag).d("element : $word")
+
+                    when (word) {
+                        "학생증" -> {
+                            viewModel.setIsStudent(true)
+                            isSuccess = true
+                        }
+                        "청소년증" -> {
+                            viewModel.setIsStudent(false)
+                            isSuccess = true
+                        }
+                        else -> {
+                            if (isSuccess && viewModel.idName.value == "" && word.length == 3) viewModel.setIdName(word)
+                            if (isSuccess && viewModel.idName.value == "" && word.startsWith("명:")) viewModel.setIdName(word.substring(2, 5))
+                            if (isSuccess && viewModel.isStudentId.value == true && viewModel.idSubEntry.value == "" && word.endsWith("학교")) viewModel.setIdSchool(word)
+                            if (isSuccess && viewModel.isStudentId.value == false && viewModel.idSubEntry.value == "" && word.length == 14 && word.contains('-')) viewModel.setBirthDate(word)
+                        }
+                    }
+                }
+            }
+        }
+
+        if (isSuccess) {
             val ocrResultBottomSheet = OcrResultFragment()
             ocrResultBottomSheet.show(supportFragmentManager, ocrResultBottomSheet.tag)
         }
@@ -86,4 +182,8 @@ class OcrActivity : BindingActivity<ActivityOcrBinding>(R.layout.activity_ocr) {
         super.onDestroy()
         cameraExecutor.shutdown()
     }
+
+    companion object {
+        private const val tag = "OCR_TEST"
+    }
 }

app/src/main/java/com/keyneez/presentation/ocr/OcrViewModel.kt

@@ -16,9 +16,22 @@ class OcrViewModel @Inject constructor() : ViewModel() {
     val isPassive: LiveData<Boolean>
         get() = _isPassive
 
+    private val _idName = MutableLiveData<String>()
+    val idName: MutableLiveData<String>
+        get() = _idName
+
+    private val _idSubEntry = MutableLiveData<String>()
+    val idSubEntry: MutableLiveData<String>
+        get() = _idSubEntry
+
+    private val _isStudentId = MutableLiveData<Boolean>()
+    val isStudentId: LiveData<Boolean>
+        get() = _isStudentId
+
     init {
         _isVertical.value = false
         _isPassive.value = false
+        _isStudentId.value = true
     }
 
     /** 카메라 프레임 회전 */
@@ -30,4 +43,37 @@ class OcrViewModel @Inject constructor() : ViewModel() {
     fun updateRecognitionType() {
         _isPassive.value = !requireNotNull(_isPassive.value)
     }
+
+    /** 신분증 이름 설정 */
+    fun setIdName(name: String) {
+        _idName.value = name
+    }
+
+    /** 신분증 학교 설정 */
+    fun setIdSchool(school: String) {
+        _idSubEntry.value = school
+    }
+
+    /** 생년월일 설정 */
+    fun setBirthDate(date: String) {
+        // date 형식 : 000000-0000000
+        _idSubEntry.value = date.substring(0, 6)
+    }
+
+    /** 학생증 여부 판단 */
+    fun setIsStudent(isStudent: Boolean) {
+        _isStudentId.value = isStudent
+    }
+
+    /** ID 카드 유형 변경 */
+    fun updateIdType(isStudent: Boolean) {
+        _isStudentId.value = isStudent
+        _idSubEntry.value = ""
+    }
+
+    /** 텍스트 추출 초기화 */
+    fun resetIdInfo() {
+        _idName.value = ""
+        _idSubEntry.value = ""
+    }
 }

app/src/main/java/com/keyneez/presentation/ocr/dialog/OcrResultFragment.kt

@@ -4,7 +4,7 @@ import android.app.Activity.RESULT_OK
 import android.content.Intent
 import android.os.Bundle
 import android.view.View
-import androidx.fragment.app.viewModels
+import com.keyneez.presentation.ocr.OcrActivity
 import com.keyneez.presentation.ocr.guide.OcrGuideActivity
 import com.keyneez.util.binding.BindingBottomSheetDialog
 import com.keyneez.util.extension.hideKeyboard
@@ -14,11 +14,9 @@ import com.lab.keyneez.databinding.BotSheetOcrResultBinding
 
 class OcrResultFragment :
     BindingBottomSheetDialog<BotSheetOcrResultBinding>(R.layout.bot_sheet_ocr_result) {
-    private val viewModel by viewModels<OcrResultViewModel>()
-
     override fun onViewCreated(view: View, savedInstanceState: Bundle?) {
         super.onViewCreated(view, savedInstanceState)
-        binding.vm = viewModel
+        binding.vm = (activity as OcrActivity).viewModel
 
         initHideKeyboard()
         initReshootBtnClickListener()

app/src/main/java/com/keyneez/presentation/signup/danal/guide/DanalGuideFragment.kt

@@ -33,6 +33,9 @@ class DanalGuideFragment :
         binding.layoutDanalGuide.setOnSingleClickListener {
             requireActivity().hideKeyboard(requireView())
         }
+        binding.layoutDanalGuideContent.setOnSingleClickListener {
+            requireActivity().hideKeyboard(requireView())
+        }
     }
 
     private fun initBackBtnClickListener() {

app/src/main/java/com/keyneez/util/binding/BindingAdapter.kt

@@ -15,8 +15,6 @@ object BindingAdapter {
     @BindingAdapter("setRoundedImage")
     fun ImageView.setRoundedImage(url: String?) {
         this.load(url) {
-            fallback(R.drawable.img_like_background)
-            placeholder(R.drawable.img_like_background)
             transformations(RoundedCornersTransformation(14f))
         }
     }