Skip to content

feat: support StarCoder model architectures #3187

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 30 commits into from
Sep 15, 2023
Merged

feat: support StarCoder model architectures #3187

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
0c5d4d8
add placeholder of starcoder in gguf / llama.cpp
wsxiaoys Sep 15, 2023
eb7f0eb
support convert starcoder weights to gguf
wsxiaoys Sep 15, 2023
76d32cc
convert MQA to MHA
wsxiaoys Sep 15, 2023
7e0a843
fix ffn_down name
wsxiaoys Sep 15, 2023
7298c37
add LLM_ARCH_STARCODER to llama.cpp
wsxiaoys Sep 15, 2023
166a259
set head_count_kv = 1
wsxiaoys Sep 15, 2023
57f064d
load starcoder weight
wsxiaoys Sep 15, 2023
a17ef39
add max_position_embeddings
wsxiaoys Sep 15, 2023
2683611
set n_positions to max_positioin_embeddings
wsxiaoys Sep 15, 2023
77c7ec1
properly load all starcoder params
wsxiaoys Sep 15, 2023
0be15e1
fix head count kv
wsxiaoys Sep 15, 2023
dac31da
fix comments
wsxiaoys Sep 15, 2023
4420cff
fix vram calculation for starcoder
wsxiaoys Sep 15, 2023
ab13d07
store mqa directly
wsxiaoys Sep 15, 2023
8bc76a2
add input embeddings handling
wsxiaoys Sep 15, 2023
101c578
add TBD
wsxiaoys Sep 15, 2023
a1cf66e
working in cpu, metal buggy
wsxiaoys Sep 15, 2023
6c353dc
cleanup useless code
wsxiaoys Sep 15, 2023
f82328a
metal : fix out-of-bounds access in soft_max kernels
ggerganov Sep 15, 2023
92a4f86
llama : make starcoder graph build more consistent with others
ggerganov Sep 15, 2023
caa7220
Merge pull request #2 from ggerganov/support-starcoder-fix
wsxiaoys Sep 15, 2023
57eaa39
refactor: cleanup comments a bit
wsxiaoys Sep 15, 2023
5ca037b
add other starcoder models: 3B, 7B, 15B
wsxiaoys Sep 15, 2023
08f35c4
support-mqa-directly
wsxiaoys Sep 15, 2023
e1fa9dd
Merge pull request #3 from TabbyML/support-starcoder-mqa
wsxiaoys Sep 15, 2023
f989ba1
fix: remove max_position_embeddings, use n_train_ctx
wsxiaoys Sep 15, 2023
bb9931c
Update llama.cpp
wsxiaoys Sep 15, 2023
eafcc34
Update llama.cpp
wsxiaoys Sep 15, 2023
e30ad71
Apply suggestions from code review
wsxiaoys Sep 15, 2023
72a7285
fix: switch to space from tab
wsxiaoys Sep 15, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
fix: remove max_position_embeddings, use n_train_ctx
  • Loading branch information
@wsxiaoys
wsxiaoys committed Sep 15, 2023
commit f989ba151d6e437a7ae45d383ffafa6653d8e7ad
3 changes: 1 addition & 2 deletions convert-starcoder-hf-to-gguf.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,9 +103,8 @@ def parse_args() -> argparse.Namespace:
block_count = hparams["n_layer"]

gguf_writer.add_name("StarCoder")
gguf_writer.add_context_length(2048) # not in config.json
gguf_writer.add_context_length(hparams["n_positions"])
gguf_writer.add_embedding_length(hparams["n_embd"])
gguf_writer.add_max_position_embeddings(hparams["n_positions"])
gguf_writer.add_feed_forward_length(4 * hparams["n_embd"])
gguf_writer.add_block_count(block_count)
gguf_writer.add_head_count(hparams["n_head"])
Expand Down
17 changes: 6 additions & 11 deletions gguf-py/gguf/gguf.py
View file
Open in desktop
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@monatis Do we need to bump gguf.py version after this change?

Original file line number Diff line number Diff line change
Expand Up @@ -36,13 +36,12 @@
KEY_GENERAL_FILE_TYPE = "general.file_type"

# LLM
KEY_CONTEXT_LENGTH = "{arch}.context_length"
KEY_EMBEDDING_LENGTH = "{arch}.embedding_length"
KEY_BLOCK_COUNT = "{arch}.block_count"
KEY_FEED_FORWARD_LENGTH = "{arch}.feed_forward_length"
KEY_USE_PARALLEL_RESIDUAL = "{arch}.use_parallel_residual"
KEY_TENSOR_DATA_LAYOUT = "{arch}.tensor_data_layout"
KEY_MAX_POSITION_EMBEDDINGS = "{arch}.max_position_embeddings"
KEY_CONTEXT_LENGTH = "{arch}.context_length"
KEY_EMBEDDING_LENGTH = "{arch}.embedding_length"
KEY_BLOCK_COUNT = "{arch}.block_count"
KEY_FEED_FORWARD_LENGTH = "{arch}.feed_forward_length"
KEY_USE_PARALLEL_RESIDUAL = "{arch}.use_parallel_residual"
KEY_TENSOR_DATA_LAYOUT = "{arch}.tensor_data_layout"

# attention
KEY_ATTENTION_HEAD_COUNT = "{arch}.attention.head_count"
Expand Down Expand Up @@ -718,10 +717,6 @@ def add_embedding_length(self, length: int):
self.add_uint32(
KEY_EMBEDDING_LENGTH.format(arch=self.arch), length)

def add_max_position_embeddings(self, length: int):
self.add_uint32(
KEY_MAX_POSITION_EMBEDDINGS.format(arch=self.arch), length)

def add_block_count(self, length: int):
self.add_uint32(
KEY_BLOCK_COUNT.format(arch=self.arch), length)
Expand Down
6 changes: 1 addition & 5 deletions llama.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -193,7 +193,6 @@ enum llm_kv {
LLM_KV_FEED_FORWARD_LENGTH,
LLM_KV_USE_PARALLEL_RESIDUAL,
LLM_KV_TENSOR_DATA_LAYOUT,
LLM_KV_MAX_POSITION_EMBEDDINGS,

LLM_KV_ATTENTION_HEAD_COUNT,
LLM_KV_ATTENTION_HEAD_COUNT_KV,
Expand Down Expand Up @@ -238,7 +237,6 @@ static std::map<llm_kv, std::string> LLM_KV_NAMES = {
{ LLM_KV_FEED_FORWARD_LENGTH, "%s.feed_forward_length" },
{ LLM_KV_USE_PARALLEL_RESIDUAL, "%s.use_parallel_residual" },
{ LLM_KV_TENSOR_DATA_LAYOUT, "%s.tensor_data_layout" },
{ LLM_KV_MAX_POSITION_EMBEDDINGS, "%s.max_position_embeddings" },

{ LLM_KV_ATTENTION_HEAD_COUNT, "%s.attention.head_count" },
{ LLM_KV_ATTENTION_HEAD_COUNT_KV, "%s.attention.head_count_kv" },
Expand Down Expand Up @@ -940,7 +938,6 @@ struct llama_hparams {
uint32_t n_layer = 32;
uint32_t n_rot = 64;
uint32_t n_ff = 11008;
uint32_t n_positions = 0; // StarCoder

float f_norm_eps = 1e-5;
float f_norm_rms_eps = 1e-5;
Expand Down Expand Up @@ -1668,7 +1665,6 @@ static void llm_load_hparams(
GGUF_GET_KEY(ctx, hparams.n_ff, gguf_get_val_u32, GGUF_TYPE_UINT32, true, kv(LLM_KV_FEED_FORWARD_LENGTH));
GGUF_GET_KEY(ctx, hparams.n_head, gguf_get_val_u32, GGUF_TYPE_UINT32, true, kv(LLM_KV_ATTENTION_HEAD_COUNT));
GGUF_GET_KEY(ctx, hparams.n_layer, gguf_get_val_u32, GGUF_TYPE_UINT32, true, kv(LLM_KV_BLOCK_COUNT));
GGUF_GET_KEY(ctx, hparams.n_positions, gguf_get_val_u32, GGUF_TYPE_UINT32, true, kv(LLM_KV_MAX_POSITION_EMBEDDINGS));

// n_head_kv is optional, default to n_head
hparams.n_head_kv = hparams.n_head;
Expand Down Expand Up @@ -2215,7 +2211,7 @@ static void llm_load_tensors(
case LLM_ARCH_STARCODER:
{
model.tok_embeddings = ml.create_tensor(ctx, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab}, GGML_BACKEND_CPU);
model.pos_embeddings = ml.create_tensor(ctx, tn(LLM_TENSOR_POS_EMBD, "weight"), {n_embd, hparams.n_positions}, GGML_BACKEND_CPU);
model.pos_embeddings = ml.create_tensor(ctx, tn(LLM_TENSOR_POS_EMBD, "weight"), {n_embd, hparams.n_ctx_train}, GGML_BACKEND_CPU);

// output
{
Expand Down
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy