From 31d578ff3b46bb4e3ef7b911328f7dce8a21a26b Mon Sep 17 00:00:00 2001 From: Matt Watson Date: Fri, 16 Feb 2024 16:35:19 -0800 Subject: [PATCH] Update API docs for KerasNLP 0.8 release --- requirements.txt | 2 +- scripts/autogen.py | 2 +- scripts/nlp_api_master.py | 172 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 174 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index ca8940241e..7c98c10c32 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,4 +14,4 @@ tensorflow keras-tuner==1.4.6 keras-cv==0.8.2 tf_keras==2.15.0 -keras-nlp==0.7.0 +keras-nlp==0.8.0 diff --git a/scripts/autogen.py b/scripts/autogen.py index efc82a18d7..682f65ed9f 100644 --- a/scripts/autogen.py +++ b/scripts/autogen.py @@ -47,7 +47,7 @@ "keras": f"{KERAS_TEAM_GH}/keras/tree/v3.0.5/", "keras_tuner": f"{KERAS_TEAM_GH}/keras-tuner/tree/v1.4.6/", "keras_cv": f"{KERAS_TEAM_GH}/keras-cv/tree/v0.8.2/", - "keras_nlp": f"{KERAS_TEAM_GH}/keras-nlp/tree/v0.7.0/", + "keras_nlp": f"{KERAS_TEAM_GH}/keras-nlp/tree/v0.8.0/", "tf_keras": f"{KERAS_TEAM_GH}/tf-keras/tree/v2.15.0/", } USE_MULTIPROCESSING = False diff --git a/scripts/nlp_api_master.py b/scripts/nlp_api_master.py index 47855182d0..5ebfbebc48 100644 --- a/scripts/nlp_api_master.py +++ b/scripts/nlp_api_master.py @@ -65,6 +65,62 @@ }, ], }, + + { + "path": "bart/", + "title": "Bart", + "toc": True, + "children": [ + { + "path": "bart_tokenizer", + "title": "BertTokenizer", + "generate": [ + "keras_nlp.models.BertTokenizer", + "keras_nlp.models.BertTokenizer.from_preset", + ], + }, + { + "path": "bart_preprocessor", + "title": "BertPreprocessor layer", + "generate": [ + "keras_nlp.models.BertPreprocessor", + "keras_nlp.models.BertPreprocessor.from_preset", + "keras_nlp.models.BertPreprocessor.tokenizer", + ], + }, + { + "path": "bart_backbone", + "title": "BertBackbone model", + "generate": [ + "keras_nlp.models.BertBackbone", + "keras_nlp.models.BertBackbone.from_preset", + "keras_nlp.models.BertBackbone.token_embedding", + ], + }, + { + "path": "bart_seq_2_seq_lm", + "title": "BartSeq2SeqLM model", + "generate": [ + "keras_nlp.models.BartSeq2SeqLM", + "keras_nlp.models.BartSeq2SeqLM.from_preset", + "keras_nlp.models.BartSeq2SeqLM.generate", + "keras_nlp.models.BartSeq2SeqLM.backbone", + "keras_nlp.models.BartSeq2SeqLM.preprocessor", + ], + }, + { + "path": "bart_seq_2_seq_lm_preprocessor", + "title": "BartSeq2SeqLMPreprocessor layer", + "generate": [ + "keras_nlp.models.BartSeq2SeqLMPreprocessor", + "keras_nlp.models.BartSeq2SeqLMPreprocessor.from_preset", + "keras_nlp.models.BartSeq2SeqLMPreprocessor.generate_preprocess", + "keras_nlp.models.BartSeq2SeqLMPreprocessor.generate_postprocess", + "keras_nlp.models.BartSeq2SeqLMPreprocessor.tokenizer", + ], + }, + ], + }, { "path": "bert/", "title": "Bert", @@ -251,6 +307,63 @@ }, ], }, + + { + "path": "gemma/", + "title": "Gemma", + "toc": True, + "children": [ + { + "path": "gemma_tokenizer", + "title": "GemmaTokenizer", + "generate": [ + "keras_nlp.models.GemmaTokenizer", + "keras_nlp.models.GemmaTokenizer.from_preset", + ], + }, + { + "path": "gemma_preprocessor", + "title": "GemmaPreprocessor layer", + "generate": [ + "keras_nlp.models.GemmaPreprocessor", + "keras_nlp.models.GemmaPreprocessor.from_preset", + "keras_nlp.models.GemmaPreprocessor.tokenizer", + ], + }, + { + "path": "gemma_backbone", + "title": "GemmaBackbone model", + "generate": [ + "keras_nlp.models.GemmaBackbone", + "keras_nlp.models.GemmaBackbone.from_preset", + "keras_nlp.models.GemmaBackbone.token_embedding", + "keras_nlp.models.GemmaBackbone.enable_lora", + "keras_nlp.models.GemmaBackbone.get_layout_map", + ], + }, + { + "path": "gemma_causal_lm", + "title": "GemmaCausalLM model", + "generate": [ + "keras_nlp.models.GemmaCausalLM", + "keras_nlp.models.GemmaCausalLM.from_preset", + "keras_nlp.models.GemmaCausalLM.generate", + "keras_nlp.models.GemmaCausalLM.backbone", + "keras_nlp.models.GemmaCausalLM.preprocessor", + "keras_nlp.models.GemmaCausalLM.score", + ], + }, + { + "path": "gemma_causal_lm_preprocessor", + "title": "GemmaCausalLMPreprocessor layer", + "generate": [ + "keras_nlp.models.GemmaCausalLMPreprocessor", + "keras_nlp.models.GemmaCausalLMPreprocessor.from_preset", + "keras_nlp.models.GemmaCausalLMPreprocessor.tokenizer", + ], + }, + ], + }, { "path": "gpt2/", "title": "GPT2", @@ -368,6 +481,60 @@ }, ], }, + { + "path": "mistral/", + "title": "Mistral", + "toc": True, + "children": [ + { + "path": "mistral_tokenizer", + "title": "MistralTokenizer", + "generate": [ + "keras_nlp.models.MistralTokenizer", + "keras_nlp.models.MistralTokenizer.from_preset", + ], + }, + { + "path": "mistral_preprocessor", + "title": "MistralPreprocessor layer", + "generate": [ + "keras_nlp.models.MistralPreprocessor", + "keras_nlp.models.MistralPreprocessor.from_preset", + "keras_nlp.models.MistralPreprocessor.tokenizer", + ], + }, + { + "path": "mistral_backbone", + "title": "MistralBackbone model", + "generate": [ + "keras_nlp.models.MistralBackbone", + "keras_nlp.models.MistralBackbone.from_preset", + "keras_nlp.models.MistralBackbone.token_embedding", + "keras_nlp.models.MistralBackbone.enable_lora", + ], + }, + { + "path": "mistral_causal_lm", + "title": "MistralCausalLM model", + "generate": [ + "keras_nlp.models.MistralCausalLM", + "keras_nlp.models.MistralCausalLM.from_preset", + "keras_nlp.models.MistralCausalLM.generate", + "keras_nlp.models.MistralCausalLM.backbone", + "keras_nlp.models.MistralCausalLM.preprocessor", + ], + }, + { + "path": "mistral_causal_lm_preprocessor", + "title": "MistralCausalLMPreprocessor layer", + "generate": [ + "keras_nlp.models.MistralCausalLMPreprocessor", + "keras_nlp.models.MistralCausalLMPreprocessor.from_preset", + "keras_nlp.models.MistralCausalLMPreprocessor.tokenizer", + ], + }, + ], + }, { "path": "opt/", "title": "OPT", @@ -774,6 +941,11 @@ "title": "TokenAndPositionEmbedding layer", "generate": ["keras_nlp.layers.TokenAndPositionEmbedding"], }, + { + "path": "alibi_bias", + "title": "AlibiBias layer", + "generate": ["keras_nlp.layers.AlibiBias"], + }, { "path": "masked_lm_head", "title": "MaskedLMHead layer",