github-starred/gitea
2
0
Fork 0
mirror of https://github.com/go-gitea/gitea.git synced 2026-03-09 12:46:42 -05:00
Code Issues 2.6k Packages Projects Releases 100 Wiki Activity

Gitea goes reproducibly "out of memory" on a big repository #6269

New Issue
Closed
opened 2025-11-02 06:50:31 -06:00 by GiteaMirror · 7 comments
GiteaMirror commented 2025-11-02 06:50:31 -06:00
Owner
Copy Link

Originally created by @blacky14 on GitHub (Nov 7, 2020).

  • Gitea version (or commit ref): 1.12.5
  • Git version: 2.24.3 (inside of docker container)
  • Operating system:
    • docker container from docker hub (Linux docker_gitea 4.9.0-13-amd64 #1 SMP Debian 4.9.228-1 (2020-07-05) x86_64 Linux)
    • running on a DigitalOcean Droplet with 1GB memory
  • Database (use [x]):
    • PostgreSQL
    • MySQL
    • MSSQL
    • SQLite
  • Can you reproduce the bug at https://try.gitea.io:
    • Yes (provide example URL)
    • No
  • Log gist:
fatal error: runtime: out of memory

runtime stack:
runtime.throw(0x231dd45, 0x16)
        /usr/local/go/src/runtime/panic.go:1116 +0x72
runtime.sysMap(0xc024000000, 0xc000000, 0x5614cb8)
        /usr/local/go/src/runtime/mem_linux.go:169 +0xc5
runtime.(*mheap).sysAlloc(0x55fe940, 0x9400000, 0x55fe948, 0x4970)
        /usr/local/go/src/runtime/malloc.go:715 +0x1cd
runtime.(*mheap).grow(0x55fe940, 0x4970, 0x0)
        /usr/local/go/src/runtime/mheap.go:1286 +0x11c
runtime.(*mheap).allocSpan(0x55fe940, 0x4970, 0x440100, 0x5614cc8, 0xfffffffffffffade)
        /usr/local/go/src/runtime/mheap.go:1124 +0x6a0
runtime.(*mheap).alloc.func1()
        /usr/local/go/src/runtime/mheap.go:871 +0x64
runtime.(*mheap).alloc(0x55fe940, 0x4970, 0x7fdbed500001, 0x446af5)
        /usr/local/go/src/runtime/mheap.go:865 +0x81
runtime.largeAlloc(0x92e0000, 0xc000040100, 0xc0032e2c00)
        /usr/local/go/src/runtime/malloc.go:1152 +0x92
runtime.mallocgc.func1()
        /usr/local/go/src/runtime/malloc.go:1047 +0x46
runtime.systemstack(0x0)
        /usr/local/go/src/runtime/asm_amd64.s:370 +0x66
runtime.mstart()
        /usr/local/go/src/runtime/proc.go:1056

goroutine 1546 [running]:
runtime.systemstack_switch()
        /usr/local/go/src/runtime/asm_amd64.s:330 fp=0xc0061f34d0 sp=0xc0061f34c8 pc=0x46d590
runtime.mallocgc(0x92e0000, 0x0, 0x0, 0xc0061f35e8)
        /usr/local/go/src/runtime/malloc.go:1046 +0x895 fp=0xc0061f3570 sp=0xc0061f34d0 pc=0x414835
runtime.growslice(0x1e28e60, 0xc012d26000, 0x757fff1, 0x7580000, 0x7587ff1, 0xc0083e8000, 0x8000, 0x8000)
        /usr/local/go/src/runtime/slice.go:175 +0x14e fp=0xc0061f35d8 sp=0xc0061f3570 pc=0x45469e
github.com/go-git/go-git/v5/plumbing.(*MemoryObject).Write(0xc007b167c0, 0xc0083e8000, 0x8000, 0x8000, 0x8000, 0x0, 0x0)
        /go/src/code.gitea.io/gitea/vendor/github.com/go-git/go-git/v5/plumbing/memory.go:53 +0xfd fp=0xc0061f3640 sp=0xc0061f35d8 pc=0x7901cd
io.copyBuffer(0x7fdbed2c4c00, 0xc007b167c0, 0x3f43c20, 0xc006ec0030, 0xc0083e8000, 0x8000, 0x8000, 0x0, 0x1be6289906202bb9, 0x1146e36193b0f1e7)
        /usr/local/go/src/io/io.go:407 +0x1fb fp=0xc0061f36b8 sp=0xc0061f3640 pc=0x4b3ebb
io.Copy(...)
        /usr/local/go/src/io/io.go:364
github.com/go-git/go-git/v5/storage/filesystem.(*ObjectStorage).getFromUnpacked(0xc00306b378, 0xf4bacd8a82962d20, 0xe1da147cd6c0edb4, 0x15b4ce6a, 0x3f83340, 0xc007b167c0, 0x0, 0x0)
        /go/src/code.gitea.io/gitea/vendor/github.com/go-git/go-git/v5/storage/filesystem/object.go:415 +0x4d7 fp=0xc0061f3868 sp=0xc0061f36b8 pc=0x8a0377
github.com/go-git/go-git/v5/storage/filesystem.(*ObjectStorage).EncodedObject(0xc00306b378, 0xbacd8a82962d2003, 0xda147cd6c0edb4f4, 0x15b4ce6ae1, 0x0, 0x0, 0x0, 0x0)
        /go/src/code.gitea.io/gitea/vendor/github.com/go-git/go-git/v5/storage/filesystem/object.go:318 +0x3ac fp=0xc0061f3908 sp=0xc0061f3868 pc=0x89fbbc
github.com/go-git/go-git/v5/plumbing/object.GetBlob(0x3f7ef00, 0xc00306b360, 0xf4bacd8a82962d20, 0xe1da147cd6c0edb4, 0x15b4ce6a, 0x82962d20000081a4, 0xd6c0edb4f4bacd8a, 0x15b4ce6ae1da147c)
        /go/src/code.gitea.io/gitea/vendor/github.com/go-git/go-git/v5/plumbing/object/blob.go:23 +0x4e fp=0xc0061f3958 sp=0xc0061f3908 pc=0x82374e
github.com/go-git/go-git/v5/plumbing/object.(*FileIter).Next(0xc00825fb88, 0x0, 0x0, 0x0)
        /go/src/code.gitea.io/gitea/vendor/github.com/go-git/go-git/v5/plumbing/object/file.go:100 +0xdd fp=0xc0061f3a08 sp=0xc0061f3958 pc=0x82d63d
github.com/go-git/go-git/v5/plumbing/object.(*FileIter).ForEach(0xc0061f3b88, 0xc0061f3b28, 0x0, 0x0)
        /go/src/code.gitea.io/gitea/vendor/github.com/go-git/go-git/v5/plumbing/object/file.go:116 +0x61 fp=0xc0061f3a70 sp=0xc0061f3a08 pc=0x82d7f1
code.gitea.io/gitea/modules/git.(*Repository).GetLanguageStats(0xc00a3debd0, 0xc004869c80, 0x28, 0xc004869c00, 0x28, 0x0)
        /go/src/code.gitea.io/gitea/modules/git/repo_language_stats.go:60 +0x37f fp=0xc0061f3bf8 sp=0xc0061f3a70 pc=0xa2b02f
code.gitea.io/gitea/modules/indexer/stats.(*DBIndexer).Index(0x5612450, 0xa, 0x0, 0x0)
        /go/src/code.gitea.io/gitea/modules/indexer/stats/db.go:49 +0x203 fp=0xc0061f3c80 sp=0xc0061f3bf8 pc=0x16c0253
code.gitea.io/gitea/modules/indexer/stats.handle(0xc007a12200, 0x1, 0x1)
        /go/src/code.gitea.io/gitea/modules/indexer/stats/queue.go:23 +0x9b fp=0xc0061f3d00 sp=0xc0061f3c80 pc=0x16c0b3b
code.gitea.io/gitea/modules/queue.NewChannelUniqueQueue.func1(0xc00a3228c0, 0x1, 0x14)
        /go/src/code.gitea.io/gitea/modules/queue/unique_queue_channel.go:59 +0x13d fp=0xc0061f3d70 sp=0xc0061f3d00 pc=0x16af21d
code.gitea.io/gitea/modules/queue.(*WorkerPool).doWork(0xc003188980, 0x3f72960, 0xc003278640)
        /go/src/code.gitea.io/gitea/modules/queue/workerpool.go:383 +0x892 fp=0xc0061f3f78 sp=0xc0061f3d70 pc=0x16adca2
code.gitea.io/gitea/modules/queue.(*WorkerPool).addWorkers.func1(0xc003188980, 0x3f72960, 0xc003278640)
        /go/src/code.gitea.io/gitea/modules/queue/workerpool.go:238 +0x43 fp=0xc0061f3fc8 sp=0xc0061f3f78 pc=0x16b0023
runtime.goexit()
        /usr/local/go/src/runtime/asm_amd64.s:1373 +0x1 fp=0xc0061f3fd0 sp=0xc0061f3fc8 pc=0x46f6a1
created by code.gitea.io/gitea/modules/queue.(*WorkerPool).addWorkers
        /go/src/code.gitea.io/gitea/modules/queue/workerpool.go:237 +0x56
...

Description

I have a repository where I added quite some PDFs over the time, making it's size up to ~200MB.
recently I also added some bigger ZIPs in there, making it's current size about ~1000MB.

since then I'm experiencing recurring "out of memory" crashes when accessing this repository.
most of the time when I'm trying to access one of its markdown pages in the web interface.
the logs from above were taken from a crash that happened immediately after pushing some additional ZIP files.

I know I shouldn't add such big files to a git repo and rather use LFS for that (which I did not try yet).
anyway, is there anything I can do now?

update:
now it seems to crash every time after I push anything to this repository.
when I look at the container memory via docker stats I see the memory being in a normal range, also while pushing.
however, immediately after the push is done, the memory increases within seconds and brings down gitea.

Originally created by @blacky14 on GitHub (Nov 7, 2020). <!-- NOTE: If your issue is a security concern, please send an email to security@gitea.io instead of opening a public issue --> <!-- 1. Please speak English, this is the language all maintainers can speak and write. 2. Please ask questions or configuration/deploy problems on our Discord server (https://discord.gg/gitea) or forum (https://discourse.gitea.io). 3. Please take a moment to check that your issue doesn't already exist. 4. Please give all relevant information below for bug reports, because incomplete details will be handled as an invalid report. --> - Gitea version (or commit ref): 1.12.5 - Git version: 2.24.3 (inside of docker container) - Operating system: <!-- Please include information on whether you built gitea yourself, used one of our downloads or are using some other package --> <!-- Please also tell us how you are running gitea, e.g. if it is being run from docker, a command-line, systemd etc. ---> <!-- If you are using a package or systemd tell us what distribution you are using --> - docker container from docker hub (`Linux docker_gitea 4.9.0-13-amd64 #1 SMP Debian 4.9.228-1 (2020-07-05) x86_64 Linux`) - running on a DigitalOcean Droplet with 1GB memory - Database (use `[x]`): - [ ] PostgreSQL - [ ] MySQL - [ ] MSSQL - [x] SQLite - Can you reproduce the bug at https://try.gitea.io: - [ ] Yes (provide example URL) - [x] No - Log gist: <!-- It really is important to provide pertinent logs --> <!-- Please read https://docs.gitea.io/en-us/logging-configuration/#debugging-problems --> <!-- In addition, if your problem relates to git commands set `RUN_MODE=dev` at the top of app.ini --> ``` fatal error: runtime: out of memory runtime stack: runtime.throw(0x231dd45, 0x16) /usr/local/go/src/runtime/panic.go:1116 +0x72 runtime.sysMap(0xc024000000, 0xc000000, 0x5614cb8) /usr/local/go/src/runtime/mem_linux.go:169 +0xc5 runtime.(*mheap).sysAlloc(0x55fe940, 0x9400000, 0x55fe948, 0x4970) /usr/local/go/src/runtime/malloc.go:715 +0x1cd runtime.(*mheap).grow(0x55fe940, 0x4970, 0x0) /usr/local/go/src/runtime/mheap.go:1286 +0x11c runtime.(*mheap).allocSpan(0x55fe940, 0x4970, 0x440100, 0x5614cc8, 0xfffffffffffffade) /usr/local/go/src/runtime/mheap.go:1124 +0x6a0 runtime.(*mheap).alloc.func1() /usr/local/go/src/runtime/mheap.go:871 +0x64 runtime.(*mheap).alloc(0x55fe940, 0x4970, 0x7fdbed500001, 0x446af5) /usr/local/go/src/runtime/mheap.go:865 +0x81 runtime.largeAlloc(0x92e0000, 0xc000040100, 0xc0032e2c00) /usr/local/go/src/runtime/malloc.go:1152 +0x92 runtime.mallocgc.func1() /usr/local/go/src/runtime/malloc.go:1047 +0x46 runtime.systemstack(0x0) /usr/local/go/src/runtime/asm_amd64.s:370 +0x66 runtime.mstart() /usr/local/go/src/runtime/proc.go:1056 goroutine 1546 [running]: runtime.systemstack_switch() /usr/local/go/src/runtime/asm_amd64.s:330 fp=0xc0061f34d0 sp=0xc0061f34c8 pc=0x46d590 runtime.mallocgc(0x92e0000, 0x0, 0x0, 0xc0061f35e8) /usr/local/go/src/runtime/malloc.go:1046 +0x895 fp=0xc0061f3570 sp=0xc0061f34d0 pc=0x414835 runtime.growslice(0x1e28e60, 0xc012d26000, 0x757fff1, 0x7580000, 0x7587ff1, 0xc0083e8000, 0x8000, 0x8000) /usr/local/go/src/runtime/slice.go:175 +0x14e fp=0xc0061f35d8 sp=0xc0061f3570 pc=0x45469e github.com/go-git/go-git/v5/plumbing.(*MemoryObject).Write(0xc007b167c0, 0xc0083e8000, 0x8000, 0x8000, 0x8000, 0x0, 0x0) /go/src/code.gitea.io/gitea/vendor/github.com/go-git/go-git/v5/plumbing/memory.go:53 +0xfd fp=0xc0061f3640 sp=0xc0061f35d8 pc=0x7901cd io.copyBuffer(0x7fdbed2c4c00, 0xc007b167c0, 0x3f43c20, 0xc006ec0030, 0xc0083e8000, 0x8000, 0x8000, 0x0, 0x1be6289906202bb9, 0x1146e36193b0f1e7) /usr/local/go/src/io/io.go:407 +0x1fb fp=0xc0061f36b8 sp=0xc0061f3640 pc=0x4b3ebb io.Copy(...) /usr/local/go/src/io/io.go:364 github.com/go-git/go-git/v5/storage/filesystem.(*ObjectStorage).getFromUnpacked(0xc00306b378, 0xf4bacd8a82962d20, 0xe1da147cd6c0edb4, 0x15b4ce6a, 0x3f83340, 0xc007b167c0, 0x0, 0x0) /go/src/code.gitea.io/gitea/vendor/github.com/go-git/go-git/v5/storage/filesystem/object.go:415 +0x4d7 fp=0xc0061f3868 sp=0xc0061f36b8 pc=0x8a0377 github.com/go-git/go-git/v5/storage/filesystem.(*ObjectStorage).EncodedObject(0xc00306b378, 0xbacd8a82962d2003, 0xda147cd6c0edb4f4, 0x15b4ce6ae1, 0x0, 0x0, 0x0, 0x0) /go/src/code.gitea.io/gitea/vendor/github.com/go-git/go-git/v5/storage/filesystem/object.go:318 +0x3ac fp=0xc0061f3908 sp=0xc0061f3868 pc=0x89fbbc github.com/go-git/go-git/v5/plumbing/object.GetBlob(0x3f7ef00, 0xc00306b360, 0xf4bacd8a82962d20, 0xe1da147cd6c0edb4, 0x15b4ce6a, 0x82962d20000081a4, 0xd6c0edb4f4bacd8a, 0x15b4ce6ae1da147c) /go/src/code.gitea.io/gitea/vendor/github.com/go-git/go-git/v5/plumbing/object/blob.go:23 +0x4e fp=0xc0061f3958 sp=0xc0061f3908 pc=0x82374e github.com/go-git/go-git/v5/plumbing/object.(*FileIter).Next(0xc00825fb88, 0x0, 0x0, 0x0) /go/src/code.gitea.io/gitea/vendor/github.com/go-git/go-git/v5/plumbing/object/file.go:100 +0xdd fp=0xc0061f3a08 sp=0xc0061f3958 pc=0x82d63d github.com/go-git/go-git/v5/plumbing/object.(*FileIter).ForEach(0xc0061f3b88, 0xc0061f3b28, 0x0, 0x0) /go/src/code.gitea.io/gitea/vendor/github.com/go-git/go-git/v5/plumbing/object/file.go:116 +0x61 fp=0xc0061f3a70 sp=0xc0061f3a08 pc=0x82d7f1 code.gitea.io/gitea/modules/git.(*Repository).GetLanguageStats(0xc00a3debd0, 0xc004869c80, 0x28, 0xc004869c00, 0x28, 0x0) /go/src/code.gitea.io/gitea/modules/git/repo_language_stats.go:60 +0x37f fp=0xc0061f3bf8 sp=0xc0061f3a70 pc=0xa2b02f code.gitea.io/gitea/modules/indexer/stats.(*DBIndexer).Index(0x5612450, 0xa, 0x0, 0x0) /go/src/code.gitea.io/gitea/modules/indexer/stats/db.go:49 +0x203 fp=0xc0061f3c80 sp=0xc0061f3bf8 pc=0x16c0253 code.gitea.io/gitea/modules/indexer/stats.handle(0xc007a12200, 0x1, 0x1) /go/src/code.gitea.io/gitea/modules/indexer/stats/queue.go:23 +0x9b fp=0xc0061f3d00 sp=0xc0061f3c80 pc=0x16c0b3b code.gitea.io/gitea/modules/queue.NewChannelUniqueQueue.func1(0xc00a3228c0, 0x1, 0x14) /go/src/code.gitea.io/gitea/modules/queue/unique_queue_channel.go:59 +0x13d fp=0xc0061f3d70 sp=0xc0061f3d00 pc=0x16af21d code.gitea.io/gitea/modules/queue.(*WorkerPool).doWork(0xc003188980, 0x3f72960, 0xc003278640) /go/src/code.gitea.io/gitea/modules/queue/workerpool.go:383 +0x892 fp=0xc0061f3f78 sp=0xc0061f3d70 pc=0x16adca2 code.gitea.io/gitea/modules/queue.(*WorkerPool).addWorkers.func1(0xc003188980, 0x3f72960, 0xc003278640) /go/src/code.gitea.io/gitea/modules/queue/workerpool.go:238 +0x43 fp=0xc0061f3fc8 sp=0xc0061f3f78 pc=0x16b0023 runtime.goexit() /usr/local/go/src/runtime/asm_amd64.s:1373 +0x1 fp=0xc0061f3fd0 sp=0xc0061f3fc8 pc=0x46f6a1 created by code.gitea.io/gitea/modules/queue.(*WorkerPool).addWorkers /go/src/code.gitea.io/gitea/modules/queue/workerpool.go:237 +0x56 ... ``` ## Description I have a repository where I added quite some PDFs over the time, making it's size up to ~200MB. recently I also added some bigger ZIPs in there, making it's current size about ~1000MB. since then I'm experiencing recurring "out of memory" crashes when accessing this repository. most of the time when I'm trying to access one of its markdown pages in the web interface. the logs from above were taken from a crash that happened immediately after pushing some additional ZIP files. I know I shouldn't add such big files to a git repo and rather use LFS for that (which I did not try yet). anyway, is there anything I can do now? update: now it seems to crash every time after I push anything to this repository. when I look at the container memory via `docker stats` I see the memory being in a normal range, also while pushing. however, immediately after the push is done, the memory increases within seconds and brings down gitea.
GiteaMirror added the type/bugtype/upstream labels 2025-11-02 06:50:31 -06:00
GiteaMirror commented 2025-11-02 06:50:32 -06:00
Author
Owner
Copy Link

@hackxiu commented on GitHub (Nov 10, 2020):

I used docker to build gitea on Synology NAS, and I also found excessive memory usage. I suspect it is a memory leak. As long as you upload a large compressed package or other large files,
gitea will lead to extremely high memory usage...

@hackxiu commented on GitHub (Nov 10, 2020): I used docker to build gitea on Synology NAS, and I also found excessive memory usage. I suspect it is a memory leak. As long as you upload a large compressed package or other large files, gitea will lead to extremely high memory usage...
GiteaMirror commented 2025-11-02 06:50:33 -06:00
Author
Owner
Copy Link

@blacky14 commented on GitHub (Nov 10, 2020):

I solved the problem for my scenario, by removing the (too) large files from my repository, basically as described here: https://stackoverflow.com/a/2158271

as I'm the only one working on this repo, it wasn't a big deal to rewrite the git history.

so this at least brings my setup to a state, in which I can work with this repository in gitea, again 🙂

@blacky14 commented on GitHub (Nov 10, 2020): I solved the problem for my scenario, by removing the (too) large files from my repository, basically as described here: https://stackoverflow.com/a/2158271 as I'm the only one working on this repo, it wasn't a big deal to rewrite the git history. so this at least brings my setup to a state, in which I can work with this repository in gitea, again :slightly_smiling_face:
GiteaMirror commented 2025-11-02 06:50:33 -06:00
Author
Owner
Copy Link

@zeripath commented on GitHub (Nov 10, 2020):

There's an associated bug in go-git which is responsible for this.

Essentially there are a number of incorrect assumptions about the suitability of reading objects and pack indices directly in to the memory that go-git makes which may require significant reconsideration.

@zeripath commented on GitHub (Nov 10, 2020): There's an associated bug in go-git which is responsible for this. Essentially there are a number of incorrect assumptions about the suitability of reading objects and pack indices directly in to the memory that go-git makes which may require significant reconsideration.
GiteaMirror commented 2025-11-02 06:50:33 -06:00
Author
Owner
Copy Link

@blacky14 commented on GitHub (Nov 10, 2020):

@zeripath
thanks for the hint! 🙂
I could not find the issue you're referring to, could you please share a link to it?

@blacky14 commented on GitHub (Nov 10, 2020): @zeripath thanks for the hint! :slightly_smiling_face: I could not find the issue you're referring to, could you please share a link to it?
GiteaMirror commented 2025-11-02 06:50:33 -06:00
Author
Owner
Copy Link

@zeripath commented on GitHub (Nov 13, 2020):

https://github.com/go-git/go-git/issues/104

@zeripath commented on GitHub (Nov 13, 2020): https://github.com/go-git/go-git/issues/104
GiteaMirror commented 2025-11-02 06:50:33 -06:00
Author
Owner
Copy Link

@zeripath commented on GitHub (Nov 27, 2020):

#13673

@zeripath commented on GitHub (Nov 27, 2020): #13673
GiteaMirror commented 2025-11-02 06:50:34 -06:00
Author
Owner
Copy Link

@lunny commented on GitHub (Jan 2, 2021):

should be closed by #13673

@lunny commented on GitHub (Jan 2, 2021): should be closed by #13673
Sign in to join this conversation.
No Branch/Tag Specified
Branches Tags
Labels
Clear labels
$20
$250
$50
$500
backport/done
💎 Bounty
docs-update-needed
good first issue
hacktoberfest
issue/bounty
issue/confirmed
issue/critical
issue/duplicate
issue/needs-feedback
issue/not-a-bug
issue/regression
issue/stale
issue/workaround
lgtm/need 2
modifies/api
modifies/translation
outdated/backport/v1.18
outdated/theme/markdown
outdated/theme/timetracker
performance/bigrepo
performance/cpu
performance/memory
performance/speed
pr/breaking
proposal/accepted
proposal/rejected
pr/wip
pull-request
Mirrored from GitHub Pull Request
 reviewed/wontfix
💰 Rewarded
skip-changelog
status/blocked
topic/accessibility
topic/api
topic/authentication
topic/build
topic/code-linting
topic/commit-signing
topic/content-rendering
topic/deployment
topic/distribution
topic/federation
topic/gitea-actions
topic/issues
topic/lfs
topic/mobile
topic/moderation
topic/packages
topic/pr
topic/projects
topic/repo
topic/repo-migration
topic/security
topic/theme
topic/ui
topic/ui-interaction
topic/ux
topic/webhooks
topic/wiki
type/bug
type/deprecation
type/docs
type/enhancement
type/feature
type/miscellaneous
type/proposal
type/question
type/refactoring
type/summary
type/testing
type/upstream
No Label type/bug type/upstream
Milestone
No items
No Milestone
Projects
Clear projects
No project
Assignees
Clear assignees
GiteaMirror ninjasurge
No Assignees
1 Participants
Notifications
Due Date
No due date set.
Dependencies

No dependencies set.

Reference: github-starred/gitea#6269
Delete Branch "%!s()"

Deleting a branch is permanent. Although the deleted branch may continue to exist for a short time before it actually gets removed, it CANNOT be undone in most cases. Continue?